pilot/info/jobinfo.py

0001 # Licensed under the Apache License, Version 2.0 (the "License");
0002 # you may not use this file except in compliance with the License.
0003 # You may obtain a copy of the License at
0004 # http://www.apache.org/licenses/LICENSE-2.0
0005 #
0006 # Authors:
0007 # - Alexey Anisenkov, anisyonk@cern.ch, 2018
0008 # - Paul Nilsson, paul.nilsson@cern.ch, 2019
0009
0010
0011 """
0012 Job specific info provider mainly used to customize Queue, Site, etc data of Information Service
0013 with details fetched directly from Job instance
0014
0015 :author: Alexey Anisenkov
0016 :contact: anisyonk@cern.ch
0017 :date: January 2018
0018 """
0019
0020 import logging
0021 logger = logging.getLogger(__name__)
0022
0023
0024 class JobInfoProvider(object):
0025     """
0026         Job info provider which is used to extract settings specific for given Job
0027         and overwrite general configuration used by Information Service
0028     """
0029
0030     job = None  ## Job instance
0031
0032     def __init__(self, job):
0033         self.job = job
0034
0035     def resolve_schedconf_sources(self):
0036         """
0037             Resolve Job specific prioritized list of source names to be used for SchedConfig data load
0038             :return: prioritized list of source names
0039         """
0040
0041         ## FIX ME LATER
0042         ## quick stub implementation: extract later from jobParams, e.g. from overwriteAGISData..
0043         ## an example of return data:
0044         ## return ['AGIS', 'LOCAL', 'CVMFS']
0045         ##
0046
0047         return None  ## Not implemented yet
0048
0049     def resolve_queuedata(self, pandaqueue, **kwargs):
0050         """
0051             Resolve Job specific settings for queue data (overwriteQueueData)
0052             :return: dict of settings for given PandaQueue as a key
0053         """
0054
0055         # use following keys from job definition
0056         # keys format: [(inputkey, outputkey), inputkey2]
0057         # outputkey is the name of external source attribute
0058         keys = [('platform', 'cmtconfig')]
0059
0060         data = {}
0061         for key in keys:
0062             if not isinstance(key, (list, tuple)):
0063                 key = [key, key]
0064             ikey = key[0]
0065             okey = key[1] if len(key) > 1 else key[0]
0066             val = getattr(self.job, ikey)
0067             if val:  # ignore empty or zero values -- FIX ME LATER for integers later if need
0068                 data[okey] = val
0069
0070         data.update(self.job.overwrite_queuedata)  ## use job.overwrite_queuedata as a master source
0071
0072         logger.info('queuedata: following keys will be overwritten by Job values: %s' % data)
0073
0074         return {pandaqueue: data}
0075
0076     def resolve_storage_data(self, ddmendpoints=[], **kwargs):
0077         """
0078             Resolve Job specific settings for storage data (including data passed via --overwriteStorageData)
0079             :return: dict of settings for requested DDMEndpoints with ddmendpoin as a key
0080         """
0081
0082         data = {}
0083
0084         ## use job.overwrite_storagedata as a master source
0085         master_data = self.job.overwrite_storagedata or {}
0086         try:
0087             data.update((k, v) for k, v in master_data.iteritems() if k in set(ddmendpoints or master_data) & set(master_data))  # Python 2
0088         except Exception:
0089             data.update((k, v) for k, v in list(master_data.items()) if k in set(ddmendpoints or master_data) & set(master_data))  # Python 3
0090
0091         if data:
0092             logger.info('storagedata: following data extracted from Job definition will be used: %s' % data)
0093
0094         return data