user/atlas/jobdata.py

0001 # Licensed under the Apache License, Version 2.0 (the "License");
0002 # you may not use this file except in compliance with the License.
0003 # You may obtain a copy of the License at
0004 # http://www.apache.org/licenses/LICENSE-2.0
0005 #
0006 # Authors:
0007 # - Paul Nilsson, paul.nilsson@cern.ch, 2021
0008
0009 import re
0010
0011 import logging
0012 logger = logging.getLogger(__name__)
0013
0014
0015 def jobparams_prefiltering(value):
0016     """
0017     Perform pre-filtering of raw job parameters to avoid problems with especially quotation marks.
0018     The function can extract some fields from the job parameters to be put back later after actual filtering.
0019
0020     E.g. ' --athenaopts "HITtoRDO:--nprocs=$ATHENA_CORE_NUMBER" ' will otherwise become
0021     ' --athenaopts 'HITtoRDO:--nprocs=$ATHENA_CORE_NUMBER' ' which will prevent the environmental variable to be unfolded.
0022
0023     :param value: job parameters (string).
0024     :return: dictionary of fields excluded from job parameters (dictionary), updated job parameters (string).
0025     """
0026
0027     exclusions = {}
0028     pattern = re.compile(r' (\-\-athenaopts\ \"?\'?[^"]+\"?\'?)')
0029     result = re.findall(pattern, value)
0030     if result:
0031         exclusions['TOBEREPLACED1'] = result[0]
0032         # remove zip map from final jobparams
0033         value = re.sub(pattern, ' TOBEREPLACED1 ', value)  # do not remove the space
0034
0035     # add more items to the exclusions as necessary
0036
0037     logger.debug('exclusions = %s', str(exclusions))
0038     return exclusions, value
0039
0040
0041 def jobparams_postfiltering(value, exclusions={}):
0042     """
0043     Perform post-filtering of raw job parameters.
0044     Any items in the optional exclusion list will be added (space separated) at the end of the job parameters.
0045
0046     :param value: job parameters (string).
0047     :param optional exclusion: exclusion dictionary from pre-filtering function (dictinoary).
0048     :return: updated job parameters (string).
0049     """
0050
0051     for item in exclusions:
0052         value = value.replace(item, exclusions[item])
0053
0054     return value