monitor_app/tests/test_regulation.py

0001 from datetime import datetime, timedelta, timezone
0002 from django.test import SimpleTestCase
0003 from monitor_app.comp_snapshot import effective_waiting_limit, _pilot_supply_part
0004 from monitor_app.segfaults import matching_failure_finding
0005
0006
0007 class RegulationTests(SimpleTestCase):
0008     def test_known_memory_failure_does_not_exempt_another_error_or_stage(self):
0009         rules = [{'fid': 'f-15', 'memory': True, 'exit_code': 137, 'stage': 'simu'}]
0010         self.assertEqual(matching_failure_finding(rules, memory=True, exit_code=137, stage='simulation'), 'f-15')
0011         self.assertIsNone(matching_failure_finding(rules, memory=False, exit_code=78, stage='simulation'))
0012         self.assertIsNone(matching_failure_finding(rules, memory=True, exit_code=137, stage='reconstruction'))
0013
0014     def test_quiet_adjuster_retains_limit_but_report_must_be_fresh(self):
0015         now = datetime.now(timezone.utc)
0016         old = {'queue_limit': 200, 'at': (now - timedelta(hours=2)).isoformat()}
0017         self.assertEqual(effective_waiting_limit({'n_queue_limit_worker_ratio': '0',
0018                           'n_queue_limit_worker': '600'}, old, now), 600)
0019         limit = effective_waiting_limit({'n_queue_limit_worker_ratio': '30'}, old, now)
0020         self.assertEqual(limit, 200)
0021         p = {'report_age_seconds': 10, 'waiting': 200, 'limit': limit,
0022              'limit_source': 'adjuster'}
0023         part = _pilot_supply_part(p, 1000, 10, 10)
0024         self.assertEqual(part['level'], 'green')
0025         self.assertIn('last applied limit 200', part['note'])
0026         p['report_age_seconds'] = 901
0027         self.assertEqual(_pilot_supply_part(p, 1000, 10, 10)['level'], 'amber')
0028         self.assertIsNone(effective_waiting_limit({}, {}, now))
0029         self.assertIsNone(effective_waiting_limit({}, dict(old, queue_limit=-1), now))
0030         self.assertEqual(_pilot_supply_part(None, 1000, 0, 0)['level'], 'amber')
0031
0032     def test_a_queue_no_harvester_reports_reads_panda_workers(self):
0033         # NERSC's harvester sends no report: one running worker with a job
0034         # queued behind it is the one-node design, not a supply failure.
0035         part = _pilot_supply_part(None, 1, 0, 0, workers=(1, 0))
0036         self.assertEqual(part['level'], 'green')
0037         self.assertIn('1 worker(s) running', part['note'])
0038         self.assertEqual(_pilot_supply_part(None, 1, 0, 0, workers=(0, 0))['level'], 'amber')
0039         self.assertEqual(_pilot_supply_part(None, 0, 0, 0)['level'], 'grey')
0040
0041     def test_failed_count_or_stale_demand_cannot_be_green(self):
0042         p = {'report_age_seconds': 1, 'waiting': 600, 'limit': 600,
0043              'error': 'worker count query failed'}
0044         self.assertEqual(_pilot_supply_part(p, 1000, 10, 10)['level'], 'amber')
0045         p.update(error=None, demand_age_seconds=120, demand_max_age_seconds=90)
0046         self.assertEqual(_pilot_supply_part(p, 1000, 10, 10)['level'], 'amber')