source/eic-shell/install.py

0001 #!/usr/bin/env python3
0002
0003 ## eic_container: Argonne Universal EIC Container
0004
0005 '''
0006 Deploy the singularity container built by the CI for this version of the software.
0007
0008 The current version is determined from the currently loaded git branch or tag,
0009 unless it is explicitly set on the command line.
0010
0011 Authors:
0012     - Whitney Armstrong <warmstrong@anl.gov>
0013     - Sylvester Joosten <sjoosten@anl.gov>
0014 '''
0015
0016 import os
0017 import argparse
0018 import re
0019 import urllib.request
0020
0021 ## Gitlab group and project/program name.
0022 DEFAULT_IMG='eic_xl'
0023 DEFAULT_VERSION='3.0.1'
0024
0025 SHORTCUTS = ['eic-shell']
0026
0027 ## URL for the current container (git tag will be filled in by the script)
0028 ## components:
0029 ##  - {ref}:
0030 ##      - branch/tag --> git branch or tag
0031 ##      - MR XX      --> refs/merge-requests/XX/head
0032 ##      - nightly    --> just use fallback singularity pull
0033 ##  - {img}: image name
0034 ##  - {job}: the CI job that built the artifact
0035 CONTAINER_URL = r'https://eicweb.phy.anl.gov/api/v4/projects/290/jobs/artifacts/{ref}/raw/build/{img}.sif?job={job}'
0036
0037 ## Docker ref is used as fallback in case regular artifact download fails
0038 ## The components are:
0039 ## - {img}: image name
0040 ## - {tag}: docker tag associated with image
0041 ##      - master        --> testing
0042 ##      - branch/tag    --> branch/tag without leading v
0043 ##      - MR XX         --> unstable (may be incorrect if multiple MRs active)
0044 ##      - nightly       --> nightly
0045 DOCKER_REF = r'docker://eicweb/{img}:{tag}'
0046
0047 ## Singularity bind directive
0048 BIND_DIRECTIVE= '-B {0}:{0}'
0049
0050 class UnknownVersionError(Exception):
0051     pass
0052 class ContainerDownloadError(Exception):
0053     pass
0054 class InvalidArgumentError(Exception):
0055     pass
0056
0057 def smart_mkdir(dir):
0058     '''functions as mkdir -p, with a write-check.
0059
0060     Raises an exception if the directory is not writeable.
0061     '''
0062     if not os.path.exists(dir):
0063         try:
0064             os.makedirs(dir)
0065         except Exception as e:
0066             print('ERROR: unable to create directory', dir)
0067             raise e
0068     if not os.access(dir, os.W_OK):
0069         print('ERROR: We do not have the write privileges to', dir)
0070         raise InvalidArgumentError()
0071
0072 ## generic launcher bash script to launch the application
0073 _LAUNCHER='''#!/usr/bin/env bash
0074
0075 ## Boilerplate to make pipes work
0076 piped_args=
0077 if [ -p /dev/stdin ]; then
0078   # If we want to read the input line by line
0079   while IFS= read line; do
0080     if [ -z "$piped_args" ]; then
0081       piped_args="${{line}}"
0082     else
0083       piped_args="${{piped_args}}\n${{line}}"
0084     fi
0085   done
0086 fi
0087
0088 ## Fire off the application wrapper
0089 if [ ${{piped_args}} ]  ; then
0090     echo -e ${{piped_args}} | singularity exec {bind} {container} {exe} $@
0091 else
0092     singularity exec {bind} {container} {exe} $@
0093 fi
0094 '''
0095
0096 def _write_script(path, content):
0097     print(' - creating', path)
0098     with open(path, 'w') as file:
0099         file.write(content)
0100     os.system('chmod +x {}'.format(path))
0101
0102 def make_launcher(app, container, bindir,
0103                   bind='', exe=None):
0104     '''Configure and install a launcher.
0105
0106     Generic launcher script to launch applications in this container.
0107
0108     The launcher script calls the desired executable from the singularity image.
0109     As the new images have the environment properly setup, we can accomplish this
0110     without using any wrapper scripts.
0111
0112     Arguments:
0113         - app: our application
0114         - container: absolute path to container
0115         - bindir: absolute launcher install path
0116     Optional:
0117         - bind: singularity bind directives
0118         - exe: executable to be associated with app.
0119                Default is app.
0120         - env: environment directives to be added to the wrapper.
0121                Multiline string. Default is nothing
0122     '''
0123     if not exe:
0124         exe = app
0125
0126     ## paths
0127     launcher_path = '{}/{}'.format(bindir, app)
0128
0129     ## scripts --> use absolute path for wrapper path inside launcher
0130     launcher = _LAUNCHER.format(container=container,
0131                                 bind=bind,
0132                                 exe=exe)
0133
0134     ## write our scripts
0135     _write_script(launcher_path, launcher)
0136
0137 ## Generic module file
0138 _MODULEFILE='''#%Module1.0#####################################################################
0139 ##
0140 ## for {name} {version}
0141 ##
0142 proc ModulesHelp {{ }} {{
0143     puts stderr "This module sets up the environment for the {name} container"
0144 }}
0145 module-whatis "{name} {version}"
0146
0147 # For Tcl script use only
0148 set version 4.1.4
0149
0150 prepend-path    PATH    {bindir}
0151 '''
0152
0153 def make_modulefile(project, version, moduledir, bindir):
0154     '''Configure and install a modulefile for this project.
0155
0156     Arguments:
0157         - project: project name
0158         - version: project version
0159         - moduledir: root modulefile directory
0160         - bindir: where executables for this project are located
0161     '''
0162
0163     ## create our modulefile
0164     content = _MODULEFILE.format(name=project, version=version, bindir=bindir)
0165     fname = '{}/{}'.format(moduledir, version)
0166     print(' - creating', fname)
0167     with open(fname, 'w') as file:
0168         file.write(content)
0169
0170 if __name__ == "__main__":
0171     parser = argparse.ArgumentParser()
0172     parser.add_argument(
0173             'prefix',
0174             help='Install prefix. This is where the container will be deployed.')
0175     parser.add_argument(
0176             '-c', '--container',
0177             dest='container',
0178             default=DEFAULT_IMG,
0179             help='(opt.) Container to install. '
0180                  'D: {} (also available: eic_dev, and legacy "eic" container).')
0181     parser.add_argument(
0182             '-v', '--version',
0183             dest='version',
0184 #            default=project_version(),
0185             default=DEFAULT_VERSION,
0186             help='(opt.) project version. '
0187                  'D: {}. For MRs, use mr-XXX.'.format(DEFAULT_VERSION))
0188     parser.add_argument(
0189             '-f', '--force',
0190             action='store_true',
0191             help='Force-overwrite already downloaded container',
0192             default=False)
0193     parser.add_argument(
0194             '-b', '--bind-path',
0195             dest='bind_paths',
0196             action='append',
0197             help='(opt.) extra bind paths for singularity.')
0198     parser.add_argument(
0199             '-m', '--module-path',
0200             dest='module_path',
0201             help='(opt.) Root module path to install a modulefile. '
0202                  'D: Do not install a modulefile')
0203
0204     args = parser.parse_args()
0205
0206     print('Deploying', args.container, 'version', args.version)
0207
0208     ## Check if our bind paths are valid
0209     bind_directive = ''
0210     if args.bind_paths and len(args.bind_paths):
0211         print('Singularity bind paths:')
0212         for path in args.bind_paths:
0213             print(' -', path)
0214             if not os.path.exists(path):
0215                 print('ERROR: path', path, 'does not exist.')
0216                 raise InvalidArgumentError()
0217         bind_directive = ' '.join([BIND_DIRECTIVE.format(path) for path in args.bind_paths])
0218
0219     ## Naming schemes:
0220     ## We need to deduce both the correct git branch and an appropriate
0221     ## local version number from the desired version number
0222     ## by default we use whatever version number is given in VERSION, but we want
0223     ## to allow users to specify either X.Y.Z or vX.Y.Z for versions (same for stable
0224     ## branches).
0225     ##
0226     ## Policy:
0227     ## numbered releases: (v)X.Y.Z --> git vX.Y.Z and local X.Y.Z
0228     ## stable branches: (v)X.Y-stable --> git vX.Y-stable and local X.Y-stable
0229     ## master branch: latest/master --> git master and local stable
0230     ## for other branches --> git <BRANCH> and local unstable
0231
0232     version_docker = None
0233     version_gitlab = None
0234     build_job = '{}:singularity:default'.format(args.container)
0235     ## firs look for spacialty containers
0236     if args.container == 'acts_material_scan':
0237         version_docker = args.version
0238         version_gitlab = 'acts-material-scan' #dashes, not underscores
0239     elif args.version in ('master', 'testing'):
0240         version_docker = 'testing'
0241         version_gitlab = 'master'
0242     elif re.search('[0-9]+\.[0-9]', args.version) is not None:
0243         suffix='-stable'
0244         if re.search('{}$'.format(suffix), args.version):
0245             suffix=''
0246         version_docker = args.version + suffix
0247         version_gitlab = args.version + suffix
0248         if version_docker[0] == 'v':
0249             version_docker = version_docker[1:]
0250         if version_gitlab[0].isdigit():
0251             version_gitlab = 'v{}'.format(version_gitlab)
0252     elif args.version[:3] == 'mr-':
0253         version_docker = 'unstable'
0254         version_gitlab = 'refs/merge-requests/{}/head'.format(args.version[3:])
0255     elif args.version == 'nightly':
0256         version_docker = 'nightly'
0257         version_gitlab = 'master'
0258         build_job = '{}:singularity:nightly'.format(args.container)
0259     else:
0260         ## fixme add proper error handling
0261         print('Unknown requested version:', args.version)
0262         raise UnknownVersionError()
0263
0264     ## 'master' is always docker-tagged as testing
0265     if version_docker == 'master':
0266         version_docker = testing
0267
0268     ## when working with the old container, the build job is just 'singularity'
0269     if args.container == 'eic':
0270         build_job = 'singularity'
0271
0272     ## Create our install prefix if needed and ensure it is writable
0273     args.prefix = os.path.abspath(args.prefix)
0274     if not args.module_path:
0275         deploy_local=True
0276     else:
0277         deploy_local=False
0278     print('Install prefix:', args.prefix)
0279     print('Creating install prefix if needed...')
0280     bindir = '{}/bin'.format(args.prefix)
0281     libdir = '{}/lib'.format(args.prefix)
0282     libexecdir = '{}/libexec'.format(args.prefix)
0283     root_prefix = os.path.abspath('{}/..'.format(args.prefix))
0284     dirs = [bindir, libdir, libexecdir]
0285     if not deploy_local:
0286         moduledir = '{}/{}'.format(args.module_path, args.container)
0287         dirs.append(moduledir)
0288     for dir in dirs:
0289         print(' -', dir)
0290         smart_mkdir(dir)
0291
0292     ## At this point we know we can write to our desired prefix and that we have a set of
0293     ## valid bind paths
0294
0295     ## Get the container
0296     ## We want to slightly modify our version specifier: if it leads with a 'v' drop the v
0297     img = args.container
0298     ## Builder SIF is not built anymore, deprecated
0299     #if args.builder:
0300         #img += "_builder"
0301     container = '{}/{}-{}.sif'.format(libdir, img, version_docker)
0302     if not os.path.exists(container) or args.force:
0303         url = CONTAINER_URL.format(ref=version_gitlab, img=img, job=build_job)
0304         print('Downloading container from:', url)
0305         print('Destination:', container)
0306         try:
0307             urllib.request.urlretrieve(url, container)
0308         except:
0309             print('WARNING: failed to retrieve container artifact')
0310             print('Attempting alternative download from docker registry')
0311             cmd = ['singularity pull', '--force', container, DOCKER_REF.format(img=img, tag=version_docker)]
0312             cmd = ' '.join(cmd)
0313             print('Executing:', cmd)
0314             err = os.system(cmd)
0315             if err:
0316                 raise ContainerDownloadError()
0317     else:
0318         print('WARNING: Container found at', container)
0319         print(' ---> run with -f to force a re-download')
0320
0321     if not deploy_local:
0322         make_modulefile(args.container, version_docker, moduledir, bindir)
0323
0324     ## configure the application launchers
0325     print('Configuring applications launchers: ')
0326     for prog in SHORTCUTS:
0327         app = prog
0328         exe = prog
0329         if type(prog) == tuple:
0330             app = prog[0]
0331             exe = prog[1]
0332         make_launcher(app, container, bindir,
0333                       bind=bind_directive,
0334                       exe=exe)
0335
0336     print('Container deployment successful!')