pandaharvester/harvesterzipper/dummy_zipper.py

0001 import uuid
0002
0003 from pandaharvester.harvestercore import core_utils
0004
0005 from .base_zipper import BaseZipper
0006
0007 # logger
0008 _logger = core_utils.setup_logger("dummy_zipper")
0009
0010
0011 # dummy plugin for zipper
0012 class DummyZipper(BaseZipper):
0013     # constructor
0014     def __init__(self, **kwarg):
0015         BaseZipper.__init__(self, **kwarg)
0016
0017     # zip output files
0018     def zip_output(self, jobspec):
0019         """Zip output files. This method loops over jobspec.outFiles, which is a list of zip file's FileSpecs,
0020         to make a zip file for each zip file's FileSpec. FileSpec.associatedFiles is a list of FileSpecs of
0021         associated files to be zipped. The path of each associated file is available in associated
0022         file's FileSpec.path. Once zip files are made, their FileSpec.path, FileSpec.fsize and
0023         FileSpec.chksum need to be set.
0024
0025         :param jobspec: job specifications
0026         :type jobspec: JobSpec
0027         :return: A tuple of return code (True for success, False otherwise) and error dialog
0028         :rtype: (bool, string)
0029         """
0030         # make logger
0031         tmpLog = self.make_logger(_logger, f"PandaID={jobspec.PandaID}", method_name="zip_output")
0032         return self.simple_zip_output(jobspec, tmpLog)
0033
0034     # asynchronous zip output
0035     def async_zip_output(self, jobspec):
0036         """Zip output files asynchronously. This method is followed by post_zip_output(),
0037         which is typically useful to trigger an asynchronous zipping mechanism such as batch job.
0038         This method loops over jobspec.outFiles, which is a list of zip file's FileSpecs, to make
0039         a zip file for each zip file's FileSpec. FileSpec.associatedFiles is a list of FileSpecs
0040         of associated files to be zipped. The path of each associated file is available in associated
0041         file's FileSpec.path.
0042
0043         :param jobspec: job specifications
0044         :type jobspec: JobSpec
0045         :return: A tuple of return code (True for success, False for fatal, None for pending) and error dialog
0046         :rtype: (bool, string)
0047         """
0048         # make logger
0049         tmpLog = self.make_logger(_logger, f"PandaID={jobspec.PandaID}", method_name="async_zip_output")
0050         tmpLog.debug("start")
0051         # set some ID which can be used for lookup in post_zip_output()
0052         groupID = str(uuid.uuid4())
0053         lfns = []
0054         for fileSpec in jobspec.outFiles:
0055             lfns.append(fileSpec.lfn)
0056         jobspec.set_groups_to_files({groupID: {"lfns": lfns, "groupStatus": "zipping"}})
0057         return True, ""
0058
0059     # post zipping
0060     def post_zip_output(self, jobspec):
0061         """This method is executed after async_zip_output(), to do post-processing for zipping.
0062         Once zip files are made, this method needs to look over jobspec.outFiles to set their
0063         FileSpec.path, FileSpec.fsize, and FileSpec.chksum.
0064
0065         :param jobspec: job specifications
0066         :type jobspec: JobSpec
0067         :return: A tuple of return code (True for success, False for fatal, None for pending) and error dialog
0068         :rtype: (bool, string)
0069         """
0070         # make logger
0071         tmpLog = self.make_logger(_logger, f"PandaID={jobspec.PandaID}", method_name="post_zip_output")
0072         tmpLog.debug("start")
0073         # get groups for lookup
0074         groups = jobspec.get_groups_of_output_files()
0075         # do something with groupIDs
0076         pass
0077         # update file attributes
0078         for fileSpec in jobspec.outFiles:
0079             fileSpec.path = "/path/to/zip"
0080             fileSpec.fsize = 12345
0081             fileSpec.chksum = "66bb0985"
0082         return True, ""