pilot/test/test_jobreport_parser.py

0001 #!/usr/bin/env python
0002 # Licensed under the Apache License, Version 2.0 (the "License");
0003 # you may not use this file except in compliance with the License.
0004 # You may obtain a copy of the License at
0005 # http://www.apache.org/licenses/LICENSE-2.0
0006 #
0007 # Authors:
0008 # - Paul Nilsson, paul.nilsson@cern.ch
0009
0010 import unittest
0011 import json
0012
0013 from pilot.user.atlas.common import parse_jobreport_data
0014
0015
0016 class TestUtils(unittest.TestCase):
0017     """
0018     Unit tests for utils functions.
0019     """
0020
0021     def setUp(self):
0022         # skip tests if running on a Mac -- Macs don't have /proc
0023         #self.mac = False
0024         # if os.environ.get('MACOSX') == 'true':
0025         #    self.mac = True
0026
0027         #from pilot.info import infosys
0028         # infosys.init("AGLT2_TEST-condor")
0029         pass
0030
0031     def test_failed_jobreport(self):
0032         """
0033         ..
0034
0035         :return: (assertion)
0036         """
0037
0038         report = """
0039         {
0040   "cmdLine": "'/ccs/proj/csc108/AtlasReleases/21.0.15/AtlasOffline/21.0.15/InstallArea/x86_64-slc6-gcc49-opt/share/Sim_tf.py'""" \
0041             + """ '--inputEVNTFile=EVNT.13276676._000045.pool.root.1' '--maxEvents=50' '--postInclude' 'default:RecJobTransforms/UseFrontier.py' """ \
0042             + """'--preExec' """ \
0043             + """'EVNTtoHITS:simFlags.SimBarcodeOffset.set_Value_and_Lock(200000)' 'EVNTtoHITS:simFlags.TRTRangeCut=30.0;simFlags.TightMuonStepping=True' """ \
0044             + """'--preInclude' 'EVNTtoHITS:SimulationJobOptions/preInclude.BeamPipeKill.py,SimulationJobOptions/preInclude.FrozenShowersFCalOnly.py' """ \
0045             + """'--skipEvents=9250' '--firstEvent=449251' '--outputHITSFile=HITS.13287270._195329.pool.root.1' '--physicsList=FTFP_BERT_ATL_VALIDATION' """ \
0046             + """'--randomSeed=8986' '--conditionsTag' 'default:OFLCOND-MC16-SDR-14' '--geometryVersion=default:ATLAS-R2-2016-01-00-01_VALIDATION' """ \
0047             + """'--runNumber=364177' '--AMITag=s3126' '--DataRunNumber=284500' '--simulator=FullG4' '--truthStrategy=MC15aPlus'",
0048   "created": "2018-03-04T09:33:09",
0049   "executor": [
0050     {
0051       "errMsg": null,
0052       "exeConfig": {
0053         "script": "athena.py",
0054         "substep": "sim"
0055       },
0056       "name": "EVNTtoHITS",
0057       "rc": -1,
0058       "statusOK": false,
0059       "validation": false
0060     }
0061   ],
0062   "exitAcronym": "TRF_EXEC_VALIDATION_FAIL",
0063   "exitCode": 66,
0064   "exitMsg": "File EVNT.13276676._000045.pool.root.1 did not pass corruption test",
0065   "exitMsgExtra": "",
0066   "files": {
0067     "input": [
0068       {
0069         "dataset": null,
0070         "nentries": null,
0071         "subFiles": [
0072           {
0073             "file_guid": null,
0074             "name": "EVNT.13276676._000045.pool.root.1"
0075           }
0076         ],
0077         "type": "EVNT"
0078       }
0079     ],
0080     "output": [
0081       {
0082         "argName": "outputHITSFile",
0083         "dataset": null,
0084         "subFiles": [
0085           {
0086             "file_guid": null,
0087             "file_size": null,
0088             "name": "HITS.13287270._195329.pool.root.1",
0089             "nentries": null
0090           }
0091         ],
0092         "type": "HITS"
0093       }
0094     ]
0095   },
0096   "name": "Sim_tf",
0097   "reportVersion": "2.0.7",
0098   "resource": {
0099     "executor": {
0100       "EVNTtoHITS": {
0101         "cpuTime": null,
0102         "postExe": {
0103           "cpuTime": null,
0104           "wallTime": null
0105         },
0106         "preExe": {
0107           "cpuTime": null,
0108           "wallTime": null
0109         },
0110         "total": {
0111           "cpuTime": null,
0112           "wallTime": null
0113         },
0114         "validation": {
0115           "cpuTime": null,
0116           "wallTime": null
0117         },
0118         "wallTime": null
0119       }
0120     },
0121     "machine": {
0122       "cpu_family": "21",
0123       "linux_distribution": [
0124         "SUSE Linux Enterprise Server ",
0125         "11",
0126         "x86_64"
0127       ],
0128       "model": "1",
0129       "model_name": "AMD Opteron(TM) Processor 6274",
0130       "node": "nid15407",
0131       "platform": "Linux-3.0.101-0.46.1_1.0502.8871-cray_gem_c-x86_64-with-SuSE-11-x86_64"
0132     },
0133     "transform": {
0134       "cpuTime": 2,
0135       "cpuTimeTotal": 0,
0136       "externalCpuTime": 0,
0137       "trfPredata": null,
0138       "wallTime": 0
0139     }
0140   }
0141 }
0142         """
0143         report_data = json.loads(report)
0144         print((json.dumps(parse_jobreport_data(report_data), sort_keys=True, indent=2)))
0145
0146     def test_successful_jobreport(self):
0147         """
0148         ..
0149
0150         :return: (assertion)
0151         """
0152
0153         report = """
0154 {
0155   "cmdLine": "'/cvmfs/atlas.cern.ch/repo/sw/software/21.0/AtlasOffline/21.0.15/InstallArea/x86_64-slc6-gcc49-opt/share/Sim_tf.py'""" \
0156             + """'--inputEVNTFile=EVNT.08847247._002212.pool.root.1' ",
0157   "executor": [
0158     {
0159       "asetup": null,
0160       "errMsg": "",
0161       "exeConfig": {
0162         "inputs": [
0163           "EVNT"
0164         ],
0165         "outputs": [
0166           "HITS"
0167         ],
0168         "script": "athena.py",
0169         "substep": "sim"
0170       },
0171       "logfileReport": {
0172         "countSummary": {
0173           "CATASTROPHE": 0,
0174           "CRITICAL": 0,
0175           "DEBUG": 0,
0176           "ERROR": 0,
0177           "FATAL": 0,
0178           "IGNORED": 0,
0179           "INFO": 11748,
0180           "UNKNOWN": 270546,
0181           "VERBOSE": 0,
0182           "WARNING": 169
0183         },
0184         "details": {
0185           "WARNING": [
0186             {
0187               "count": 1,
0188               "firstLine": 10898,
0189               "message": "-------- WWWW ------- G4Exception-START -------- WWWW -------\\n""" \
0190             + """*** G4Exception : GeomNav1002\\nissued by : G4Navigator::ComputeStep()\\nTrack stuck or not moving.\\n""" \
0191             + """Track stuck, not moving for 10 steps\\nin volume -LArMgr::LAr::EMEC::Neg::InnerWheel::Lead- at point (-509.71,377.051,-3897.05)\\n""" \
0192             + """direction: (0.259253,0.546694,-0.796186).\\nPotential geometry or navigation problem !\\n""" \
0193             + """Trying pushing it of 1e-07 mm ...Potential overlap in geometry!\\n\\n*** This is just a warning message. ***\\n""" \
0194             + """-------- WWWW -------- G4Exception-END --------- WWWW -------"
0195             } ]
0196         }
0197       },
0198       "metaData": {},
0199       "name": "EVNTtoHITS",
0200       "rc": 0,
0201       "statusOK": true,
0202       "validation": true
0203     }
0204   ],
0205   "exitAcronym": "OK",
0206   "exitCode": 0,
0207   "exitMsg": "OK",
0208   "exitMsgExtra": "",
0209   "files": {
0210     "input": [
0211       {
0212         "dataset": null,
0213         "nentries": 5000,
0214         "subFiles": [
0215           {
0216             "file_guid": "A7AD81C0-B1B0-D341-A229-D8E7B066EF08",
0217             "name": "EVNT.08847247._002212.pool.root.1"
0218           }
0219         ],
0220         "type": "EVNT"
0221       }
0222     ],
0223     "output": [
0224       {
0225         "argName": "outputHITSFile",
0226         "dataset": null,
0227         "subFiles": [
0228           {
0229             "file_guid": "9051A804-2965-9346-9ED7-64435E702DFC",
0230             "file_size": 156541511,
0231             "name": "HITS.12039077._013137.pool.root.1",
0232             "nentries": 250
0233           }
0234         ],
0235         "type": "HITS"
0236       }
0237     ]
0238   },
0239   "name": "Sim_tf",
0240   "reportVersion": "2.0.7",
0241   "resource": {
0242     "dbDataTotal": 4284032,
0243     "dbTimeTotal": 9.74,
0244     "executor": {
0245       "EVNTtoHITS": {
0246         "cpuTime": 243302,
0247         "cpuTimePerWorker": 1900.796875,
0248         "dbData": 4251604,
0249         "dbTime": 7.29,
0250         "memory": {
0251           "Avg": {
0252             "avgPSS": 6536986,
0253             "avgRSS": 129252048,
0254             "avgSwap": 0,
0255             "avgVMEM": 179846244,
0256             "rateRBYTES": 165880,
0257             "rateRCHAR": 450275,
0258             "rateWBYTES": 70567,
0259             "rateWCHAR": 83780
0260           },
0261           "Max": {
0262             "maxPSS": 12599089,
0263             "maxRSS": 252762140,
0264             "maxSwap": 0,
0265             "maxVMEM": 349917308,
0266             "totRBYTES": 676790272,
0267             "totRCHAR": 1837125486,
0268             "totWBYTES": 287916032,
0269             "totWCHAR": 341826635
0270           }
0271         },
0272         "mpworkers": 128,
0273         "nevents": 250,
0274         "postExe": {
0275           "cpuTime": 1036,
0276           "wallTime": 306
0277         },
0278         "preExe": {
0279           "cpuTime": 0,
0280           "wallTime": 0
0281         },
0282         "total": {
0283           "cpuTime": 244337,
0284           "wallTime": 5917
0285         },
0286         "validation": {
0287           "cpuTime": 0,
0288           "wallTime": 13
0289         },
0290         "wallTime": 5598
0291       },
0292       "HITSMergeAthenaMP0": {
0293         "cpuTime": 298,
0294         "dbData": 32428,
0295         "dbTime": 2.45,
0296         "memory": {
0297           "Avg": {
0298             "avgPSS": 574755,
0299             "avgRSS": 722394,
0300             "avgSwap": 0,
0301             "avgVMEM": 1366161,
0302             "rateRBYTES": 13980,
0303             "rateRCHAR": 1754533,
0304             "rateWBYTES": 413340,
0305             "rateWCHAR": 439546
0306           },
0307           "Max": {
0308             "maxPSS": 802907,
0309             "maxRSS": 1156932,
0310             "maxSwap": 0,
0311             "maxVMEM": 2418240,
0312             "totRBYTES": 4194304,
0313             "totRCHAR": 526360001,
0314             "totWBYTES": 124002304,
0315             "totWCHAR": 131863967
0316           }
0317         },
0318         "postExe": {
0319           "cpuTime": 1,
0320           "wallTime": 1
0321         },
0322         "preExe": {
0323           "cpuTime": 0,
0324           "wallTime": 3
0325         },
0326         "total": {
0327           "cpuTime": 299,
0328           "wallTime": 305
0329         },
0330         "validation": {
0331           "cpuTime": 0,
0332           "wallTime": 1
0333         },
0334         "wallTime": 301
0335       }
0336     },
0337     "machine": {
0338       "cpu_family": "6",
0339       "linux_distribution": [
0340         "CentOS",
0341         "6.9",
0342         "Final"
0343       ],
0344       "model": "87",
0345       "model_name": "Intel(R) Xeon Phi(TM) CPU 7230 @ 1.30GHz",
0346       "node": "nid00056",
0347       "platform": "Linux-4.4.49-92.11.1_3.0-cray_ari_c-x86_64-with-centos-6.9-Final"
0348     },
0349     "transform": {
0350       "cpuEfficiency": 0.32,
0351       "cpuPWEfficiency": 0.4074,
0352       "cpuTime": 226,
0353       "cpuTimeTotal": 244636,
0354       "externalCpuTime": 738,
0355       "processedEvents": 250,
0356       "trfPredata": null,
0357       "wallTime": 5952
0358     }
0359   }
0360 }
0361         """
0362         report_data = json.loads(report)
0363         print((json.dumps(parse_jobreport_data(report_data), sort_keys=True, indent=2)))
0364
0365
0366 if __name__ == '__main__':
0367     unittest.main()