user/atlas/dbrelease.py

0001 #!/usr/bin/env python
0002 # Licensed under the Apache License, Version 2.0 (the "License");
0003 # you may not use this file except in compliance with the License.
0004 # You may obtain a copy of the License at
0005 # http://www.apache.org/licenses/LICENSE-2.0
0006 #
0007 # Authors:
0008 # - Paul Nilsson, paul.nilsson@cern.ch, 2019-2021
0009
0010 import os
0011 import re
0012 import tarfile
0013 import logging
0014
0015 from pilot.common.exception import FileHandlingFailure, PilotException
0016 from pilot.util.filehandling import write_file, mkdirs, rmdirs
0017
0018 logger = logging.getLogger(__name__)
0019
0020
0021 def extract_version(name):
0022     """
0023     Try to extract the version from the DBRelease string.
0024
0025     :param name: DBRelease (string).
0026     :return: version (string).
0027     """
0028     version = ""
0029
0030     re_v = re.compile(r'DBRelease-(\d+\.\d+\.\d+)\.tar\.gz')  # Python 3 (added r)
0031     ver = re_v.search(name)
0032     if ver:
0033         version = ver.group(1)
0034     else:
0035         re_v = re.compile(r'DBRelease-(\d+\.\d+\.\d+\.\d+)\.tar\.gz')  # Python 3 (added r)
0036         ver = re_v.search(name)
0037         if ver:
0038             version = ver.group(1)
0039
0040     return version
0041
0042
0043 def get_dbrelease_version(jobpars):
0044     """
0045     Get the DBRelease version from the job parameters.
0046
0047     :param jobpars: job parameters (string).
0048     :return: DBRelease version (string).
0049     """
0050
0051     return extract_version(jobpars)
0052
0053
0054 def get_dbrelease_dir():
0055     """
0056     Return the proper DBRelease directory
0057
0058     :return: path to DBRelease (string).
0059     """
0060
0061     path = os.path.join(os.environ.get('VO_ATLAS_SW_DIR', 'OSG_APP'), 'database/DBRelease')
0062     if path == "" or path.startswith('OSG_APP'):
0063         logger.warning("note: the DBRelease database directory is not available (will not attempt to skip DBRelease stage-in)")
0064     else:
0065         if os.path.exists(path):
0066             logger.info("local DBRelease path verified: %s (will attempt to skip DBRelease stage-in)", path)
0067         else:
0068             logger.warning("note: local DBRelease path does not exist: %s (will not attempt to skip DBRelease stage-in)", path)
0069
0070     return path
0071
0072
0073 def is_dbrelease_available(version):
0074     """
0075     Check whether a given DBRelease file is already available.
0076
0077     :param version: DBRelease version (string).
0078     :return: Boolean (True is DBRelease is locally available).
0079     """
0080
0081     status = False
0082
0083     # do not proceed if
0084     if 'ATLAS_DBREL_DWNLD' in os.environ:
0085         logger.info("ATLAS_DBREL_DWNLD is set: do not skip DBRelease stage-in")
0086         return status
0087
0088     # get the local path to the DBRelease directory
0089     path = get_dbrelease_dir()
0090
0091     if path != "" and os.path.exists(path):
0092         # get the list of available DBRelease directories
0093         dir_list = os.listdir(path)
0094
0095         # is the required DBRelease version available?
0096         if dir_list:
0097             if version in dir_list:
0098                 logger.info("found version %s in path %s (%d releases found)", version, path, len(dir_list))
0099                 status = True
0100             else:
0101                 logger.warning("did not find version %s in path %s (%d releases found)", version, path, len(dir_list))
0102         else:
0103             logger.warning("empty DBRelease directory list: %s", path)
0104     else:
0105         logger.warning('no such DBRelease path: %s', path)
0106
0107     return status
0108
0109
0110 def create_setup_file(version, path):
0111     """
0112     Create the DBRelease setup file.
0113
0114     :param version: DBRelease version (string).
0115     :param path: path to local DBReleases (string).
0116     :return: Boolean (True if DBRelease setup file was successfully created).
0117     """
0118
0119     status = False
0120
0121     # get the DBRelease directory
0122     _dir = get_dbrelease_dir()
0123     if _dir != "" and version != "":
0124         # create the python code string to be written to file
0125         txt = "import os\n"
0126         txt += "os.environ['DBRELEASE'] = '%s'\n" % version
0127         txt += "os.environ['DATAPATH'] = '%s/%s:' + os.environ['DATAPATH']\n" % (_dir, version)
0128         txt += "os.environ['DBRELEASE_REQUIRED'] = '%s'\n" % version
0129         txt += "os.environ['DBRELEASE_REQUESTED'] = '%s'\n" % version
0130         txt += "os.environ['CORAL_DBLOOKUP_PATH'] = '%s/%s/XMLConfig'\n" % (_dir, version)
0131
0132         try:
0133             status = write_file(path, txt)
0134         except FileHandlingFailure as exc:
0135             logger.warning('failed to create DBRelease setup file: %s', exc)
0136         else:
0137             logger.info("Created setup file with the following content:.................................\n%s", txt)
0138             logger.info("...............................................................................")
0139     else:
0140         logger.warning('failed to create %s for DBRelease version=%s and directory=%s', path, version, _dir)
0141
0142     return status
0143
0144
0145 def create_dbrelease(version, path):
0146     """
0147     Create the DBRelease file only containing a setup file.
0148
0149     :param version: DBRelease version (string).
0150     :param path: path to DBRelease (string).
0151     :return: Boolean (True is DBRelease file was successfully created).
0152     """
0153
0154     status = False
0155
0156     # create the DBRelease and version directories
0157     dbrelease_path = os.path.join(path, 'DBRelease')
0158     _path = os.path.join(dbrelease_path, version)
0159     try:
0160         mkdirs(_path, chmod=None)
0161     except PilotException as exc:
0162         logger.warning('failed to create directories for DBRelease: %s', exc)
0163     else:
0164         logger.debug('created directories: %s', _path)
0165
0166         # create the setup file in the DBRelease directory
0167         version_path = os.path.join(dbrelease_path, version)
0168         setup_filename = "setup.py"
0169         _path = os.path.join(version_path, setup_filename)
0170         if create_setup_file(version, _path):
0171             logger.info("created DBRelease setup file: %s", _path)
0172
0173             # now create a new DBRelease tarball
0174             filename = os.path.join(path, "DBRelease-%s.tar.gz" % version)
0175             logger.info("creating file: %s", filename)
0176             try:
0177                 tar = tarfile.open(filename, "w:gz")
0178             except (IOError, OSError) as exc:
0179                 logger.warning("could not create DBRelease tar file: %s", exc)
0180             else:
0181                 if tar:
0182                     # add the setup file to the tar file
0183                     tar.add("%s/DBRelease/%s/%s" % (path, version, setup_filename))
0184
0185                     # create the symbolic link DBRelease/current ->  12.2.1
0186                     try:
0187                         _link = os.path.join(path, "DBRelease/current")
0188                         os.symlink(version, _link)
0189                     except OSError as exc:
0190                         logger.warning("failed to create symbolic link %s: %s", _link, exc)
0191                     else:
0192                         logger.warning("created symbolic link: %s", _link)
0193
0194                         # add the symbolic link to the tar file
0195                         tar.add(_link)
0196
0197                         # done with the tar archive
0198                         tar.close()
0199
0200                         logger.info("created new DBRelease tar file: %s", filename)
0201                         status = True
0202                 else:
0203                     logger.warning("failed to open DBRelease tar file")
0204
0205             # clean up
0206             if rmdirs(dbrelease_path):
0207                 logger.debug("cleaned up directories in path: %s", dbrelease_path)
0208         else:
0209             logger.warning("failed to create DBRelease setup file")
0210             if rmdirs(dbrelease_path):
0211                 logger.debug("cleaned up directories in path: %s", dbrelease_path)
0212
0213     return status