pandaserver/dataservice/closer_atlas_plugin.py

0001 """
0002 ATLAS plugin for closer
0003 """
0004 import sys
0005
0006 from pandacommon.pandalogger.LogWrapper import LogWrapper
0007 from pandaserver.dataservice.ddm import rucioAPI
0008 from pandaserver.dataservice import DataServiceUtils
0009
0010
0011 # plugin for ATLAS closer
0012 class CloserAtlasPlugin:
0013     """
0014     A class used to represent the ATLAS closer plugin.
0015
0016     ...
0017
0018     Attributes
0019     ----------
0020     jobSpec : object
0021         an object representing the job specification
0022     datasets : list
0023         a list of datasets to be processed
0024     tmp_log : LogWrapper
0025         a LogWrapper object for logging
0026
0027     Methods
0028     -------
0029     __init__(self, job, datasets, log)
0030         Constructs all the necessary attributes for the CloserAtlasPlugin object.
0031     execute(self)
0032         Executes the plugin's main functionality.
0033     """
0034
0035     # constructor
0036     def __init__(self, job, datasets, log):
0037         """
0038         Constructs all the necessary attributes for the CloserAtlasPlugin object.
0039
0040         Parameters
0041         ----------
0042             job : object
0043                 an object representing the job specification
0044             datasets : list
0045                 a list of datasets to be processed
0046             log : LogWrapper
0047                 a LogWrapper object for logging
0048         """
0049         self.jobSpec = job
0050         self.datasets = datasets
0051         self.tmp_log = LogWrapper(log, f"{self.jobSpec.PandaID} CloserAtlasPlugin")
0052
0053     # execute
0054     def execute(self):
0055         """
0056         Executes the main functionality of the CloserAtlasPlugin.
0057
0058         This method is responsible for closing datasets that meet certain criteria.
0059         It only operates on production jobs that are either 'urgent' or have a high priority.
0060         For each dataset in the job, if the dataset's name ends with '_sub' followed by a number and its status is 'tobeclosed',
0061         it attempts to close the dataset using the rucioAPI. If an error occurs during this process, it logs a warning message.
0062
0063         Returns
0064         -------
0065         bool
0066             Always returns True regardless of whether the datasets were successfully closed or not.
0067         """
0068         try:
0069             # only for production
0070             if self.jobSpec.prodSourceLabel not in ["managed", "test"]:
0071                 return True
0072             # only for urgent or high prio
0073             if self.jobSpec.processingType not in ["urgent"] and self.jobSpec.currentPriority <= 1000:
0074                 return True
0075             # close datasets
0076             for datasetSpec in self.datasets:
0077                 if not DataServiceUtils.is_sub_dataset(datasetSpec.name):
0078                     continue
0079                 if datasetSpec.status != "tobeclosed":
0080                     continue
0081                 try:
0082                     self.tmp_log.debug(f"immediate close {datasetSpec.name}")
0083                     rucioAPI.close_dataset(datasetSpec.name)
0084                 except Exception:
0085                     errtype, errvalue = sys.exc_info()[:2]
0086                     self.tmp_log.warning(f"failed to close : {errtype} {errvalue}")
0087         except Exception:
0088             errtype, errvalue = sys.exc_info()[:2]
0089             self.tmp_log.warning(f"failed to execute : {errtype} {errvalue}")
0090         return True