File indexing completed on 2026-11-09 10:47:30
0001 """The room's failures part: Snapper categories, catalog labels, the correction root."""
0002 from unittest import mock
0003
0004 from django.test import SimpleTestCase
0005
0006 from monitor_app import comp_snapshot
0007 from monitor_app.comp_snapshot import FRONT_STATES
0008
0009
0010 class _Rule:
0011 component = 'pilot'
0012 code = 1305
0013 diag_substring = ''
0014 queue_scope = ''
0015 corrected_label = 'payload failure behind a stale apptainer line'
0016 note = ''
0017 evidence_url = ''
0018
0019
0020 class TopFailuresTests(SimpleTestCase):
0021 def _builder(self):
0022 b = comp_snapshot._Build.__new__(comp_snapshot._Build)
0023 b.S = 'doma_panda'
0024 b.hour_ago = 'x'
0025 return b
0026
0027 def test_ddm_failures_are_named_not_unattributed(self):
0028 rows = [('BNL_OSG_PanDA_1', 'ddm', 200, 'Could not add files to DDM: bytes mismatch', '0', 616),
0029 ('BNL_OSG_PanDA_1', 'pilot', 1152, 'File transfer timed out', None, 2),
0030 ('UM_GREX_PanDA_1', 'none', 0, '', None, 1)]
0031 with mock.patch.object(comp_snapshot, '_rows', return_value=rows), \
0032 mock.patch.object(comp_snapshot.error_corrections, '_rules', return_value=[]):
0033 out = self._builder().top_failures()
0034 self.assertEqual(out['BNL_OSG_PanDA_1'][0], ('ddm 200 - Could not add files to DDM (registration refused)', 616))
0035 self.assertEqual(out['BNL_OSG_PanDA_1'][1][1], 2)
0036 self.assertTrue(out['BNL_OSG_PanDA_1'][1][0].startswith('pilot 1152'))
0037 self.assertEqual(out['UM_GREX_PanDA_1'], [('failed with no error code in any component', 1)])
0038 self.assertNotIn('unattributed', str(out))
0039
0040 def test_exit_codes_merge_into_one_category(self):
0041 rows = [('Q', 'executor', 78, 'registration', '78', 5), ('Q', 'executor', 78, 'registration', '78', 3)]
0042 with mock.patch.object(comp_snapshot, '_rows', return_value=rows), \
0043 mock.patch.object(comp_snapshot.error_corrections, '_rules', return_value=[]):
0044 out = self._builder().top_failures()
0045 self.assertEqual(out['Q'], [('executor 78', 8)])
0046
0047 def test_a_correction_rule_rewrites_the_label(self):
0048 rows = [('NERSC_Perlmutter_epic', 'pilot', 1305, 'bind mounting /global', '78', 40)]
0049 with mock.patch.object(comp_snapshot, '_rows', return_value=rows), \
0050 mock.patch.object(comp_snapshot.error_corrections, '_rules', return_value=[_Rule()]), \
0051 mock.patch.object(comp_snapshot.error_corrections, 'exit_reading', return_value='Rucio registration failed (exit 78)'):
0052 out = self._builder().top_failures()
0053 self.assertEqual(out['NERSC_Perlmutter_epic'], [('Rucio registration failed (exit 78)', 40)])
0054
0055 def test_a_degraded_front_holds_releases_not_the_queue(self):
0056 self.assertEqual(FRONT_STATES['degraded'], ('red', 'holds new releases, a gate is red'))
