monitor_app/tests/test_declared.py

0001 """The declared record's pure readers (monitor_app/declared.py): CRIC
0002 documents to records, a record's standing in time, the attribution
0003 window, the line a queue row shows. No store."""
0004 from datetime import datetime, timezone as dt_timezone
0005
0006 from monitor_app.declared import (
0007     declared_at, rules_from_pandaqueuestatus, rules_from_ddmendpointstatus,
0008     standing_of, summary_line, windows_from_downtime)
0009
0010
0011 # Xin's rule of 2026-09-14 as CRIC returned it (showall=1).
0012 PQS = {
0013     'E1_BNL': {'a': {
0014         'mode': {'OFFLINE': {'manual': {
0015             'activity': 'a', 'expiration': '2026-09-15T00:21:00',
0016             'last_modified': '2026-09-14T02:20:19.409339',
0017             'operationdn': 'xzhao@bnl.gov', 'pandaqueue': 'E1_BNL',
0018             'probe': 'manual', 'reason': 'scheduled downtime',
0019             'updated': '2026-09-14T02:20:19.404270', 'value': 'OFFLINE'}}},
0020         'status': {'value': 'OFFLINE'}}},
0021     'CC-IN2P3_Rubin_Merge': {'a': {'mode': {'BROKEROFF': {'manual': {
0022         'expiration': '2029-12-31T17:06:00', 'updated': '2023-05-03T19:06:54',
0023         'operationdn': 'someone', 'reason': 'manual'}}}}},
0024 }
0025
0026
0027 def _t(s):
0028     return datetime.fromisoformat(s).replace(tzinfo=dt_timezone.utc)
0029
0030
0031 def test_queue_rule_becomes_one_record_in_scope():
0032     rows = rules_from_pandaqueuestatus(PQS, {'E1_BNL', 'BNL_PanDA_1'})
0033     assert [r['name'] for r in rows] == ['queue:E1_BNL:a:OFFLINE:manual']
0034     r = rows[0]
0035     assert (r['kind'], r['target'], r['value'], r['activity']) == ('queue', 'E1_BNL', 'OFFLINE', 'a')
0036     assert r['end'] == '2026-09-15T00:21:00+00:00'
0037     assert r['declared_by'] == 'xzhao@bnl.gov' and r['reason'] == 'scheduled downtime'
0038     assert r['start'] is None
0039
0040
0041 def test_standing_follows_the_clock():
0042     r = rules_from_pandaqueuestatus(PQS, {'E1_BNL'})[0]
0043     assert standing_of(r, _t('2026-09-14T12:00:00')) == 'active'
0044     assert standing_of(r, _t('2026-09-15T00:21:00')) == 'expired'
0045     window = {'start': '2026-09-20T08:00:00+00:00', 'end': '2026-09-20T16:00:00+00:00'}
0046     assert standing_of(window, _t('2026-09-15T12:00:00')) == 'future'
0047     assert standing_of(window, _t('2026-09-20T09:00:00')) == 'active'
0048
0049
0050 def test_attribution_covers_the_window_and_the_cache_lag():
0051     r = rules_from_pandaqueuestatus(PQS, {'E1_BNL'})[0]
0052     # Zhaoyu's pilot read OFFLINE at 00:24 UTC, three minutes after expiry.
0053     assert declared_at([r], _t('2026-09-15T00:24:13'), kind='queue', target='E1_BNL')
0054     assert not declared_at([r], _t('2026-09-15T00:35:00'), kind='queue', target='E1_BNL')
0055     assert not declared_at([r], _t('2026-09-14T01:00:00'), kind='queue', target='E1_BNL')
0056     assert not declared_at([r], _t('2026-09-14T12:00:00'), kind='queue', target='E1_JLAB')
0057
0058
0059 def test_endpoint_rule_and_downtime_window():
0060     eps = rules_from_ddmendpointstatus(
0061         {'BNL_PROD_DISK_1': {'w': {'mode': {'OFF': {'manual': {
0062             'expiration': '2026-09-14T22:00:00', 'updated': '2026-09-14T02:00:00',
0063             'operationdn': 'xzhao@bnl.gov', 'reason': 'scheduled downtime'}}}}}},
0064         {'BNL_PROD_DISK_1'})
0065     assert eps[0]['name'] == 'endpoint:BNL_PROD_DISK_1:w:OFF:manual'
0066     assert eps[0]['kind'] == 'endpoint' and eps[0]['activity'] == 'w'
0067     wins = windows_from_downtime(
0068         {'123': {'id': 123, 'rc_site': 'BNL-OSG', 'severity': 'OUTAGE',
0069                  'classification': 'SCHEDULED', 'start_time': '2026-09-20T08:00:00',
0070                  'end_time': '2026-09-20T16:00:00', 'description': 'farm maintenance',
0071                  'info_url': 'https://example/x', 'affected_services': ['BNL-CE-1']},
0072          '124': {'id': 124, 'rc_site': 'CERN-PROD', 'start_time': '2026-09-20T08:00:00'}},
0073         {'BNL-OSG'})
0074     assert [w['name'] for w in wins] == ['site:BNL-OSG:123']
0075     assert wins[0]['start'] == '2026-09-20T08:00:00+00:00' and wins[0]['services'] == ['BNL-CE-1']
0076
0077
0078 def test_a_site_window_reaches_its_queues():
0079     w = windows_from_downtime(
0080         {'1': {'id': 1, 'rc_site': 'BNL-OSG', 'severity': 'OUTAGE',
0081                'start_time': '2026-09-20T08:00:00', 'end_time': '2026-09-20T16:00:00'}},
0082         {'BNL-OSG': {'BNL_OSG_PanDA_1', 'BNL_OSG_EPIC_PROD_1'}})[0]
0083     assert w['queues'] == ['BNL_OSG_EPIC_PROD_1', 'BNL_OSG_PanDA_1']
0084     assert declared_at([w], _t('2026-09-20T09:00:00'), kind='queue', target='BNL_OSG_PanDA_1')
0085     assert not declared_at([w], _t('2026-09-20T09:00:00'), kind='queue', target='E1_BNL')
0086     assert not declared_at([w], _t('2026-09-20T09:00:00'), kind='endpoint', target='BNL_OSG_PanDA_1')
0087
0088
0089 def test_node_guard_rows_under_a_declaration_are_set_aside():
0090     from monitor_app.declared import set_aside_declared
0091     r = rules_from_pandaqueuestatus(PQS, {'E1_BNL'})[0]
0092     rows = [{'queue': 'E1_BNL', 'host': 'a', 'endtime': '2026-09-14T12:00:00'},
0093             {'queue': 'E1_BNL', 'host': 'b', 'endtime': '2026-09-15T00:25:00'},
0094             {'queue': 'E1_BNL', 'host': 'c', 'endtime': '2026-09-15T02:00:00'},
0095             {'queue': 'E1_JLAB', 'host': 'd', 'endtime': '2026-09-14T12:00:00'}]
0096     kept, aside = set_aside_declared(rows, records=[r])
0097     assert [x['host'] for x in kept] == ['c', 'd']
0098     assert aside == {'E1_BNL': 2}
0099     assert set_aside_declared(rows, records=[]) == (rows, {})
0100
0101
0102 def test_the_canary_provider_answers_per_queue():
0103     from monitor_app import declared as mod
0104     r = rules_from_pandaqueuestatus(PQS, {'E1_BNL'})[0]
0105     r['standing'] = 'active'
0106     mod._cache['records'] = [r]
0107     mod._cache['at'] = float('inf')
0108     try:
0109         info = mod.canary_declared(['E1_BNL', 'E1_JLAB'], _t('2026-09-14T12:00:00'))
0110         assert set(info) == {'E1_BNL'}
0111         assert info['E1_BNL']['in_force'].startswith('offline until 09/15 00:21 UTC')
0112         assert info['E1_BNL']['spans'] == [('2026-09-14T02:20:19+00:00', '2026-09-15T00:31:00+00:00')]
0113         assert info['E1_BNL']['last_end'] == '2026-09-15T00:31:00+00:00'
0114         later = mod.canary_declared(['E1_BNL'], _t('2026-09-15T02:00:00'))
0115         assert later['E1_BNL']['in_force'] == ''
0116     finally:
0117         mod._cache['records'] = None
0118         mod._cache['at'] = 0.0
0119
0120
0121 def test_summary_lines_read_as_an_operator_would():
0122     r = rules_from_pandaqueuestatus(PQS, {'E1_BNL'})[0]
0123     assert summary_line(r) == 'offline until 09/15 00:21 UTC: scheduled downtime (xzhao@bnl.gov)'
0124     w = windows_from_downtime(
0125         {'1': {'id': 1, 'rc_site': 'BNL-OSG', 'severity': 'OUTAGE',
0126                'start_time': '2026-09-20T08:00:00', 'end_time': '2026-09-20T16:00:00',
0127                'description': 'farm maintenance', 'provider': 'gocdb'}}, {'BNL-OSG'})[0]
0128     assert summary_line(w) == 'outage 09/20 08:00 UTC to 09/20 16:00 UTC: farm maintenance (gocdb)'