Commits

Ville Suoranta authored e9814705f6d
Add script to build Python wheels for Google Colab
No tags

devbuild/build_casa_colab.py

Added
1 +import argparse
2 +import subprocess
3 +import re
4 +import logging
5 +import os
6 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
7 +
8 +parser = argparse.ArgumentParser(description='Build Casa in a container')
9 +parser.add_argument("-c", "--container", type=str, required=True,
10 + help='Source code location f.e. /home/<user>/source')
11 +parser.add_argument("-u", "--user", type=str, required=True, help='user')
12 +parser.add_argument("-d", "--uid", type=str, required=True, help='uid')
13 +parser.add_argument("--testtools", required=False, action='store_true', help='Run casatools tests')
14 +parser.add_argument("--testtasks", required=False, action='store_true', help='Run casatasks tests')
15 +parser.add_argument("-b", "--buildtarget", type=str, required=False, help='Use genmake to build casatools with. Default is bdist_wheel.')
16 +parser.add_argument("-p", "--pythonversion", type=str, required=True, help='Python version. For example 3.6')
17 +parser.add_argument('--debug', action='store_true', help='Print debug messages')
18 +
19 +args = parser.parse_args()
20 +
21 +logger = logging.getLogger()
22 +if args.debug:
23 + logger.setLevel(logging.DEBUG)
24 +
25 +logger.debug('Argument values:')
26 +logger.debug(args.container)
27 +logger.debug(args.user)
28 +logger.debug(args.uid)
29 +logger.debug(args.buildtarget)
30 +logger.debug(args.pythonversion)
31 +
32 +container = args.container
33 +user = args.user
34 +uid = args.uid
35 +pythonversion = args.pythonversion
36 +testtools = args.testtools
37 +testtasks = args.testtasks
38 +
39 +tools_target = " --relwithdebinfo --stripsyms bdist_wheel"
40 +tasks_target = "bdist_wheel"
41 +shell_target = "bdist_wheel"
42 +
43 +if args.buildtarget =="genmake":
44 + tools_target = "genmake"
45 + tasks_target = "build"
46 + shell_target = "build"
47 +
48 +mypath = os.path.dirname(os.path.abspath(__file__))
49 +logger.debug("Script path: " + mypath)
50 +
51 +def exec_as_shell(cmd):
52 + print("Executing: " + cmd)
53 + process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
54 + stderr=subprocess.STDOUT, shell=True)
55 + #stdout, stderr = process.communicate()
56 + #logger.debug(stdout)
57 + while True:
58 + output = process.stdout.readline().decode('UTF-8')
59 + if output == '' and process.poll() is not None:
60 + break
61 + if output:
62 + print (output.strip())
63 + rc = process.poll()
64 + return rc
65 +
66 +# Build a list of commands to execute
67 +cmds = []
68 +cmds.append('docker exec ' + container +' /bin/bash -c "useradd -u ' + uid + ' ' + user + '"')
69 +cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "mkdir ~/.casa && mkdir -p ~/.config/pip"')
70 +cmds.append('docker cp ' + mypath + "/config/config.py " + container +':/home/' + user +'/.casa/config.py')
71 +cmds.append('docker cp ' + mypath + "/config/pip.conf " + container +':/home/' + user +'/.config/pip/pip.conf')
72 +cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "ccache -M 50G"')
73 +cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "python'+ pythonversion + ' -m venv ~/venv" ')
74 +cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "source ~/venv/bin/activate && pip install --index-url=https://pypi.org/simple --upgrade pip wheel"')
75 +cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "source ~/venv/bin/activate && pip install --index-url=https://pypi.org/simple --upgrade packaging certifi auditwheel numpy scipy matplotlib patchelf"')
76 +cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "source ~/venv/bin/activate && cd /source/casa6/casatools/ && autoconf && ./configure && python setup.py ' + tools_target + '"')
77 +if tools_target=="genmake":
78 + cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "export CCACHE_DIR=/ccache && cd /source/casa6/casatools/ && make -j4' + '"')
79 +else:
80 + cmds.append('docker exec -u ' + user + " " + container + " /bin/bash -c 'source ~/venv/bin/activate && cd /source/casa6/casatools/dist && auditwheel repair `ls -t *.whl | head -1`'")
81 +cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "source ~/venv/bin/activate && export PYTHONPATH=/source/casa6/casatools/build/lib.linux-x86_64-'+ pythonversion + '/ && cd /source/casa6/casatasks/ && python'+ pythonversion + ' setup.py ' + tasks_target + '"')
82 +cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "source ~/venv/bin/activate && export PYTHONPATH=/source/casa6/casatools/build/lib.linux-x86_64-'+ pythonversion + '/:/source/casa6/casatasks/build/lib.linux-x86_64-'+ pythonversion + '/ && cd /source/casashell/ && python setup.py ' + tasks_target + '"')
83 +cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "mkdir /source/extralibs"')
84 +cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "cp /usr/lib64/libgfortran.so.3 /source/extralibs"')
85 +
86 +# Build casatestutils
87 +if testtools == True or testtasks != None:
88 + cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "source ~/venv/bin/activate && export PYTHONPATH=/source/casa6/casatools/build/lib.linux-x86_64-'+ pythonversion + \
89 + ':/source/casa6/casatasks/build/lib.linux-x86_64-'+ pythonversion +'/ && cd /source/casa6/casatestutils && python setup.py build"')
90 +
91 +if testtools == True:
92 + cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "source ~/venv/bin/activate && export PYTHONPATH=/source/casa6/casatools/build/lib.linux-x86_64-'+ pythonversion + \
93 + ':/source/casa6/casatestutils/build/lib'\
94 + '/ && cd /source/casa6/casatools/tests && python run.py"')
95 +if testtasks == True:
96 + cmds.append('docker exec -u ' + user + " " + container +' /bin/bash -c "source ~/venv/bin/activate && export PYTHONPATH=/source/casa6/casatools/build/lib.linux-x86_64-'+ pythonversion + \
97 + ':/source/casa6/casatestutils/build/lib'\
98 + ':/source/casa6/casatasks/build/lib.linux-x86_64-' + pythonversion +'/ && cd /source/casa6/casatasks/tests && python run.py"')
99 +
100 +# Execute commands
101 +for cmd in cmds:
102 + exec_as_shell(cmd)
103 +

Everything looks good. We'll let you know here if there's anything you should know about.

Add shortcut