pandaserver/test/daod_on_demand.py

0001 import json
0002 import math
0003 import re
0004 import sys
0005 import uuid
0006
0007 from pandajedi.jedicore import JediTaskBuffer
0008 from pandaserver.taskbuffer.Initializer import initializer
0009
0010 jediTaskID = sys.argv[1]
0011
0012 initializer.init()
0013 taskBuffer = JediTaskBuffer.JediTaskBuffer(None)
0014 proxy = taskBuffer.proxyPool.getProxy()
0015
0016 s, o = proxy.getClobObj(
0017     "select JEDI_TASK_PARAMETERS from atlas_deft.t_task where taskid=:task_id",
0018     {":task_id": jediTaskID},
0019 )
0020
0021 taskParamStr = o[0][0]
0022
0023 t = json.loads(taskParamStr)
0024
0025 s, o = proxy.querySQLS(
0026     "select cpuTime,cpuTimeUnit,ioIntensity,ioIntensityUnit,ramCount,ramUnit,outDiskCount,outDiskUnit,workDiskCount,workDiskUnit "
0027     "FROM atlas_panda.jedi_tasks where jeditaskid=:task_id",
0028     {":task_id": jediTaskID},
0029 )
0030 (
0031     cpuTime,
0032     cpuTimeUnit,
0033     ioIntensity,
0034     ioIntensityUnit,
0035     ramCount,
0036     ramUnit,
0037     outDiskCount,
0038     outDiskUnit,
0039     workDiskCount,
0040     workDiskUnit,
0041 ) = o[0]
0042
0043 tname = f"{str(uuid.uuid4())}.{jediTaskID}"
0044
0045 t.update(
0046     {
0047         "taskName": tname,
0048         "userName": "pandasrv1",
0049         "prodSourceLabel": "test",
0050         "taskPriority": 100000,
0051         "skipScout": True,
0052         "cpuTime": cpuTime,
0053         "cpuTimeUnit": cpuTimeUnit,
0054         "ioIntensity": ioIntensity,
0055         "ioIntensityUnit": ioIntensityUnit,
0056         "ramCount": ramCount,
0057         "ramUnit": ramUnit,
0058         "outDiskCount": outDiskCount,
0059         "outDiskUnit": outDiskUnit,
0060         "workDiskCount": workDiskCount,
0061         "workDiskUnit": workDiskUnit,
0062         "t1Weight": -1,
0063     }
0064 )
0065
0066 i = t["log"]
0067 i["dataset"] = f"panda.{tname}.log"
0068 i["value"] = re.sub(r".{}.".format(jediTaskID), r".{}.".format(tname), i["value"])
0069
0070 if "toStaging" in t:
0071     del t["toStaging"]
0072
0073 if "inputPreStaging" in t:
0074     del t["inputPreStaging"]
0075
0076 if "nGBPerJob" in t:
0077     t["nGBPerJob"] = math.ceil(t["nGBPerJob"] / 2)
0078
0079 newJ = []
0080 for i in t["jobParameters"]:
0081     if "dataset" in i:
0082         if i["param_type"] == "output":
0083             i["dataset"] = f"panda.{tname}.{i['dataset'].split('.')[4]}"
0084             i["value"] = re.sub(r".{}.".format(jediTaskID), r".{}.".format(tname), i["value"])
0085     elif i["type"] == "template":
0086         i["value"] = re.sub(r" {}.".format(jediTaskID), r" {}.".format(tname), i["value"])
0087
0088 k = sorted(t.keys())
0089
0090 for kk in k:
0091     print(kk, t[kk])
0092
0093 jonStr = json.dumps(t)
0094
0095 print(proxy.insertTaskParams_JEDI(t["vo"], t["prodSourceLabel"], t["userName"], t["taskName"], jonStr))