pandaserver/taskbuffer/FileSpec.py

0001 """
0002 file specification
0003
0004 """
0005
0006 import datetime
0007
0008 reserveChangedState = False
0009
0010
0011 class FileSpec(object):
0012     # attributes
0013     _attributes = (
0014         "row_ID",
0015         "PandaID",
0016         "GUID",
0017         "lfn",
0018         "type",
0019         "dataset",
0020         "status",
0021         "prodDBlock",
0022         "prodDBlockToken",
0023         "dispatchDBlock",
0024         "dispatchDBlockToken",
0025         "destinationDBlock",
0026         "destinationDBlockToken",
0027         "destinationSE",
0028         "fsize",
0029         "md5sum",
0030         "checksum",
0031         "scope",
0032         "jediTaskID",
0033         "datasetID",
0034         "fileID",
0035         "attemptNr",
0036     )
0037     # slots
0038     __slots__ = _attributes + (
0039         "_owner",
0040         "_changedAttrs",
0041         "_oldPandaID",
0042         "_reserveChangedState",
0043     )
0044     # attributes which have 0 by default
0045     _zeroAttrs = ("fsize",)
0046     # mapping between sequence and attr
0047     _seqAttrMap = {"row_ID": "ATLAS_PANDA.FILESTABLE4_ROW_ID_SEQ.nextval"}
0048
0049     # constructor
0050     def __init__(self):
0051         # install attributes
0052         for attr in self._attributes:
0053             object.__setattr__(self, attr, None)
0054         # set owner to synchronize PandaID
0055         object.__setattr__(self, "_owner", None)
0056         # map of changed attributes
0057         object.__setattr__(self, "_changedAttrs", {})
0058         # old PandaID
0059         object.__setattr__(self, "_oldPandaID", "NULL")
0060         # reserve changed state at instance level
0061         object.__setattr__(self, "_reserveChangedState", False)
0062
0063     # override __getattribute__ for SQL and PandaID
0064     def __getattribute__(self, name):
0065         # PandaID
0066         if name == "PandaID":
0067             if self._owner is None:
0068                 return "NULL"
0069             return self._owner.PandaID
0070         # others
0071         ret = object.__getattribute__(self, name)
0072         if ret is None:
0073             return "NULL"
0074         return ret
0075
0076     # override __setattr__ to collect the changed attributes
0077     def __setattr__(self, name, value):
0078         oldVal = getattr(self, name)
0079         object.__setattr__(self, name, value)
0080         newVal = getattr(self, name)
0081         # collect changed attributes
0082         if oldVal != newVal:
0083             self._changedAttrs[name] = value
0084
0085     # set owner
0086     def setOwner(self, owner):
0087         self._owner = owner
0088         self._oldPandaID = self.PandaID
0089
0090     # reset changed attribute list
0091     def resetChangedList(self):
0092         self._oldPandaID = self.PandaID
0093         object.__setattr__(self, "_changedAttrs", {})
0094
0095     # return a tuple of values
0096     def values(self):
0097         ret = []
0098         for attr in self._attributes:
0099             val = getattr(self, attr)
0100             ret.append(val)
0101         return tuple(ret)
0102
0103     # return map of values
0104     def valuesMap(self, useSeq=False, onlyChanged=False):
0105         ret = {}
0106         for attr in self._attributes:
0107             if useSeq and attr in self._seqAttrMap:
0108                 continue
0109             if onlyChanged:
0110                 if attr == "PandaID":
0111                     if self.PandaID == self._oldPandaID:
0112                         continue
0113                 elif attr not in self._changedAttrs:
0114                     continue
0115             val = getattr(self, attr)
0116             if val == "NULL":
0117                 if attr in self._zeroAttrs:
0118                     val = 0
0119                 else:
0120                     val = None
0121             ret[f":{attr}"] = val
0122         return ret
0123
0124     # pack tuple into FileSpec
0125     def pack(self, values):
0126         for i in range(len(self._attributes)):
0127             attr = self._attributes[i]
0128             val = values[i]
0129             object.__setattr__(self, attr, val)
0130
0131     # return state values to be pickled
0132     def __getstate__(self):
0133         state = []
0134         for attr in self._attributes:
0135             val = getattr(self, attr)
0136             state.append(val)
0137         if reserveChangedState or self._reserveChangedState:
0138             state.append(self._changedAttrs)
0139         # append owner info
0140         state.append(self._owner)
0141         return state
0142
0143     # restore state from the unpickled state values
0144     def __setstate__(self, state):
0145         pandaID = "NULL"
0146         for i in range(len(self._attributes)):
0147             if i + 1 < len(state):
0148                 object.__setattr__(self, self._attributes[i], state[i])
0149             else:
0150                 object.__setattr__(self, self._attributes[i], "NULL")
0151             if self._attributes[i] == "PandaID":
0152                 pandaID = state[i]
0153         object.__setattr__(self, "_owner", state[-1])
0154         object.__setattr__(self, "_oldPandaID", pandaID)
0155         if not hasattr(self, "_reserveChangedState"):
0156             object.__setattr__(self, "_reserveChangedState", False)
0157         if reserveChangedState or self._reserveChangedState:
0158             object.__setattr__(self, "_changedAttrs", state[-2])
0159         else:
0160             object.__setattr__(self, "_changedAttrs", {})
0161
0162     # return column names for INSERT
0163     def columnNames(cls, withMod=False):
0164         ret = ""
0165         for attr in cls._attributes:
0166             if ret != "":
0167                 ret += ","
0168             ret += attr
0169         # add modificationTime
0170         if withMod:
0171             ret += ",modificationTime"
0172         return ret
0173
0174     columnNames = classmethod(columnNames)
0175
0176     # return expression of values for INSERT
0177     def valuesExpression(cls):
0178         ret = "VALUES("
0179         for attr in cls._attributes:
0180             ret += "%s"
0181             if attr != cls._attributes[len(cls._attributes) - 1]:
0182                 ret += ","
0183         ret += ")"
0184         return ret
0185
0186     valuesExpression = classmethod(valuesExpression)
0187
0188     # return expression of bind variables for INSERT
0189     def bindValuesExpression(cls, useSeq=False, withMod=False):
0190         from pandaserver.config import panda_config
0191
0192         ret = "VALUES("
0193         for attr in cls._attributes:
0194             if useSeq and attr in cls._seqAttrMap:
0195                 if panda_config.backend == "mysql":
0196                     # mysql
0197                     ret += f"NULL,"
0198                 else:
0199                     # oracle
0200                     ret += f"{cls._seqAttrMap[attr]},"
0201             else:
0202                 ret += f":{attr},"
0203         ret = ret[:-1]
0204         # add modificationTime
0205         if withMod:
0206             ret += ",:modificationTime"
0207         ret += ")"
0208         return ret
0209
0210     bindValuesExpression = classmethod(bindValuesExpression)
0211
0212     # return an expression for UPDATE
0213     def updateExpression(cls):
0214         ret = ""
0215         for attr in cls._attributes:
0216             ret = ret + attr + "=%s"
0217             if attr != cls._attributes[len(cls._attributes) - 1]:
0218                 ret += ","
0219         return ret
0220
0221     updateExpression = classmethod(updateExpression)
0222
0223     # return an expression of bind variables for UPDATE
0224     def bindUpdateExpression(cls):
0225         ret = ""
0226         for attr in cls._attributes:
0227             ret += f"{attr}=:{attr},"
0228         ret = ret[:-1]
0229         ret += " "
0230         return ret
0231
0232     bindUpdateExpression = classmethod(bindUpdateExpression)
0233
0234     # return an expression of bind variables for UPDATE to update only changed attributes
0235     def bindUpdateChangesExpression(self):
0236         ret = ""
0237         for attr in self._attributes:
0238             if attr in self._changedAttrs or (attr == "PandaID" and self.PandaID != self._oldPandaID):
0239                 ret += f"{attr}=:{attr},"
0240         ret = ret[:-1]
0241         ret += " "
0242         return ret
0243
0244     # check if unmerged input
0245     def isUnMergedInput(self):
0246         if self.type == "input" and self.dispatchDBlockToken == "TOMERGE":
0247             return True
0248         return False
0249
0250     # check if unmerged output
0251     def isUnMergedOutput(self):
0252         if self.type in ["output", "log"] and self.destinationDBlockToken == "TOMERGE":
0253             return True
0254         return False
0255
0256     # allow no output
0257     def allowNoOutput(self):
0258         if self.dispatchDBlockToken in ["NULL", None, ""]:
0259             items = []
0260         else:
0261             items = self.dispatchDBlockToken.split(",")
0262         if "an" not in items:
0263             items.append("an")
0264             self.dispatchDBlockToken = ",".join(items)
0265
0266     # check if no output is allowed
0267     def isAllowedNoOutput(self):
0268         try:
0269             if "an" in self.dispatchDBlockToken.split(","):
0270                 return True
0271         except Exception:
0272             pass
0273         return False
0274
0275     # dump to be json-serializable
0276     def dump_to_json_serializable(self):
0277         stat = self.__getstate__()[:-1]
0278         # set None as _owner
0279         stat.append(None)
0280         return stat
0281
0282     # to a dictionary
0283     def to_dict(self):
0284         ret = {}
0285         for a in self._attributes:
0286             v = getattr(self, a)
0287             if v == "NULL":
0288                 v = None
0289             ret[a] = v
0290         return ret