pandaharvester/harvesterextractor/aux_extractor.py

0001 import re
0002
0003 from .base_extractor import BaseExtractor
0004
0005
0006 # extractor for auxiliary input files
0007 class AuxExtractor(BaseExtractor):
0008     # constructor
0009     def __init__(self, **kwarg):
0010         self.containerPrefix = None
0011         BaseExtractor.__init__(self, **kwarg)
0012
0013     # get auxiliary input files
0014     def get_aux_inputs(self, jobspec):
0015         url_list = []
0016         jobPars = jobspec.jobParams["jobPars"]
0017         # transformation
0018         trf = jobspec.jobParams["transformation"]
0019         if trf is not None and trf.startswith("http"):
0020             url_list.append(trf)
0021         # extract source URL
0022         tmpM = re.search(" --sourceURL\s+([^\s]+)", jobPars)
0023         if tmpM is not None:
0024             sourceURL = tmpM.group(1)
0025             jobspec.jobParams["sourceURL"] = sourceURL
0026             # extract sandbox
0027             if jobspec.jobParams["prodSourceLabel"] == "user":
0028                 tmpM = re.search("-a\s+([^\s]+)", jobPars)
0029             else:
0030                 tmpM = re.search("-i\s+([^\s]+)", jobPars)
0031             if tmpM is not None:
0032                 lfn = tmpM.group(1)
0033                 url = f"{sourceURL}/cache/{lfn}"
0034                 url_list.append(url)
0035         # extract container image
0036         if "container_name" in jobspec.jobParams:
0037             url = jobspec.jobParams["container_name"]
0038             if self.containerPrefix is not None and not url.startswith(self.containerPrefix):
0039                 url = self.containerPrefix + url
0040             url_list.append(url)
0041         return self.make_aux_inputs(url_list)