monitor_app/tests/test_epicprod_diagnosis.py

0001 from django.test import SimpleTestCase
0002
0003 from monitor_app.epicprod_inventory import diagnosis_from_log_texts
0004
0005
0006 class EpicProdDiagnosisTests(SimpleTestCase):
0007     def test_taiwan_checksum_failure_is_attributed_to_output_storage(self):
0008         stdout = """
0009 export OUT_RSE=ASGC-XRD
0010 Finished processing.
0011 ✓ VALID: /srv/job/reco.eicrecon.edm4eic.root
0012 RECO ROOT file validation passed.
0013 ERROR: Rucio registration failed for RECO file.
0014 """
0015         stderr = """
0016 Trying upload with https to EIC-XRD-LOG
0017 Successful upload of temporary file. https://dtn-rucio.jlab.org:1094/log.tar.gz
0018 Trying upload with root to ASGC-XRD
0019 Successful upload of temporary file. root://hpceph-xrootd.twgrid.org:1094/reco.root.rucio.upload
0020 RSE checksum unavailable: Timer expired
0021 Upload attempt failed
0022 Trying upload with https to ASGC-XRD
0023 Successful upload of temporary file. https://hpceph-xrootd.twgrid.org:1094/reco.root.rucio.upload
0024 RSEChecksumUnavailable: HTTP 500
0025 Upload attempt failed
0026 """
0027
0028         diagnosis = diagnosis_from_log_texts(
0029             [stdout, stderr],
0030             job={'computingsite': 'BNL_ePIC_GOOGLE', 'jobstatus': 'failed'},
0031         )
0032
0033         self.assertEqual(diagnosis['phase'], 'output_registration')
0034         self.assertTrue(diagnosis['payload_completed'])
0035         self.assertTrue(diagnosis['validation_passed'])
0036         self.assertEqual(diagnosis['operation'], 'remote_checksum')
0037         self.assertEqual(diagnosis['rse'], 'ASGC-XRD')
0038         self.assertEqual(
0039             diagnosis['endpoint'], 'hpceph-xrootd.twgrid.org')
0040         self.assertEqual(diagnosis['protocols_failed'], ['root', 'https'])
0041         self.assertEqual(diagnosis['cause_layer'], 'storage')
0042         self.assertEqual(diagnosis['cause_entity'], 'ASGC-XRD')
0043         self.assertEqual(diagnosis['cause_confidence'], 'confirmed')
0044         self.assertIn('ASGC-XRD at hpceph-xrootd.twgrid.org',
0045                       diagnosis['failure_summary'])
0046         self.assertNotIn('JLab', diagnosis['failure_summary'])
0047         self.assertNotIn('JLab', str(diagnosis['timeline']))
0048
0049     def test_duplicate_did_checksum_conflict_is_data_management(self):
0050         logs = ["""
0051 export OUT_RSE=ASGC-XRD
0052 register_to_rucio.py
0053 DataIdentifierAlreadyExists: File DID already exists
0054 Local checksum deadbeef does not match remote checksum cafebabe
0055 """]
0056
0057         diagnosis = diagnosis_from_log_texts(logs)
0058
0059         self.assertEqual(diagnosis['phase'], 'output_registration')
0060         self.assertEqual(diagnosis['operation'], 'rucio_replica_conflict')
0061         self.assertEqual(diagnosis['cause_layer'], 'data_management')
0062         self.assertEqual(diagnosis['cause_entity'], 'ASGC-XRD')
0063         self.assertEqual(diagnosis['cause_confidence'], 'confirmed')
0064         self.assertEqual(
0065             diagnosis['conflict'][1],
0066             {'local_checksum': 'deadbeef', 'remote_checksum': 'cafebabe'},
0067         )
0068
0069     def test_worker_failures_remain_separate_from_storage_failures(self):
0070         backoff = diagnosis_from_log_texts([], job={
0071             'computingsite': 'BNL_ePIC_GOOGLE',
0072             'jobstatus': 'failed',
0073             'superrordiag': 'BackoffLimitExceeded: retry limit reached',
0074         })
0075         shutdown = diagnosis_from_log_texts([], job={
0076             'computingsite': 'BNL_ePIC_GOOGLE',
0077             'jobstatus': 'failed',
0078             'piloterrordiag': 'Job killed due to imminent node shutdown',
0079         })
0080
0081         self.assertEqual(backoff['phase'], 'worker_execution')
0082         self.assertEqual(backoff['operation'], 'worker_backoff_limit')
0083         self.assertEqual(backoff['cause_layer'], 'compute')
0084         self.assertEqual(backoff['cause_entity'], 'BNL_ePIC_GOOGLE')
0085         self.assertEqual(shutdown['phase'], 'worker_execution')
0086         self.assertEqual(shutdown['operation'], 'node_shutdown')
0087         self.assertEqual(shutdown['cause_layer'], 'compute')
0088
0089     def test_top_level_checksum_without_transfer_evidence_is_unresolved(self):
0090         diagnosis = diagnosis_from_log_texts([], job={
0091             'computingsite': 'BNL_ePIC_GOOGLE',
0092             'jobstatus': 'failed',
0093             'piloterrordiag': 'Could not get the checksum',
0094         })
0095
0096         self.assertEqual(diagnosis['phase'], 'failed')
0097         self.assertEqual(diagnosis['cause_layer'], 'unknown')
0098         self.assertEqual(diagnosis['cause_entity'], '')
0099         self.assertEqual(diagnosis['cause_confidence'], 'unresolved')