swf-monitor/scripts/panda-retry-rules.py

0001 """Manage the PanDA retry-module rules for the epic instance.
0002
0003 The retry module (panda-server retryModule.py) applies error-keyed
0004 rules on top of ordinary per-attempt retries: RETRYERRORS rows match a
0005 failed job's error source, code, and diagnostic pattern and invoke a
0006 RETRYACTIONS implementation (no_retry, limit_retry, ...). A rule
0007 enforces only when BOTH switches are set: retryerrors.active = 'Y'
0008 and retryactions.active = 'Y' (db_proxy_mods/misc_standalone_module.py,
0009 getRetrialRules); otherwise the module logs what it would have done.
0010 JEDI caches the rule set, so a change takes effect within about an
0011 hour.
0012
0013 This script is the write surface for the rule-level switch
0014 (retryerrors.active) and reports both tables. The action-level switch
0015 (retryactions.active) disables an action for every rule that uses it
0016 and is deliberately not managed here. The epic rule set and its
0017 rationale: swf-epicprod docs/PANDA_ANCILLARY_AUDIT.md. The System
0018 page's PanDA Configuration section shows the same tables live.
0019
0020 Run under the venv with the swf-monitor project on the path:
0021
0022     cd <swf-monitor>/src && source <venv>/bin/activate && source ~/.env
0023     python <swf-monitor>/scripts/panda-retry-rules.py            # list
0024     python <swf-monitor>/scripts/panda-retry-rules.py --activate 1 --apply
0025     python <swf-monitor>/scripts/panda-retry-rules.py --deactivate 1 --apply
0026
0027 Dry-run without --apply: shows the switch changes it would make.
0028 """
0029
0030 import argparse
0031 import os
0032 import sys
0033
0034 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0035
0036 import django  # noqa: E402
0037
0038 django.setup()
0039
0040 from django.db import connections, transaction  # noqa: E402
0041
0042
0043 def list_rules():
0044     from monitor_app.system_status import (
0045         panda_retry_action_set,
0046         panda_retry_rule_set,
0047     )
0048     print('Rules (RETRYERRORS):')
0049     for r in panda_retry_rule_set():
0050         print(f"  id {r['id']}: {r['source']} {r['code']} "
0051               f"diag {r['diag']!r} {r['parameters']} "
0052               f"-> {r['action']}  [{r['mode']}]")
0053     print('Actions (RETRYACTIONS):')
0054     for a in panda_retry_action_set():
0055         state = 'active' if a['active'] else 'INACTIVE'
0056         print(f"  id {a['id']}: {a['action']} [{state}] {a['description']}")
0057
0058
0059 def set_rule_active(rule_ids, value, apply_changes):
0060     with connections['panda'].cursor() as cursor:
0061         placeholders = ', '.join(['%s'] * len(rule_ids))
0062         cursor.execute(
0063             f"SELECT retryerror_id, errorsource, errorcode, active"
0064             f" FROM retryerrors WHERE retryerror_id IN ({placeholders})",
0065             rule_ids)
0066         rows = {row[0]: row for row in cursor.fetchall()}
0067     missing = [i for i in rule_ids if i not in rows]
0068     if missing:
0069         print(f'ERROR: no such rule id(s): {missing}', file=sys.stderr)
0070         return 1
0071     to_change = [i for i in rule_ids if rows[i][3] != value]
0072     for rule_id in rule_ids:
0073         _, source, code, current = rows[rule_id]
0074         note = ('unchanged' if rows[rule_id][3] == value
0075                 else f"{current} -> {value}")
0076         print(f'  rule {rule_id} ({source} {code}): {note}')
0077     if not to_change:
0078         print('nothing to change')
0079         return 0
0080     if not apply_changes:
0081         print('dry run — nothing written; --apply writes the switches')
0082         return 0
0083     with transaction.atomic(using='panda'):
0084         with connections['panda'].cursor() as cursor:
0085             placeholders = ', '.join(['%s'] * len(to_change))
0086             cursor.execute(
0087                 f"UPDATE retryerrors SET active = %s"
0088                 f" WHERE retryerror_id IN ({placeholders})",
0089                 [value] + to_change)
0090             if cursor.rowcount != len(to_change):
0091                 raise RuntimeError(
0092                     f'expected {len(to_change)} rows updated, '
0093                     f'got {cursor.rowcount}')
0094     print(f'applied: {len(to_change)} rule(s) set active={value}; '
0095           f'JEDI picks this up within ~1 hour (rule cache)')
0096     return 0
0097
0098
0099 def main():
0100     parser = argparse.ArgumentParser(
0101         description='List or switch PanDA retry-module rules '
0102                     '(rule-level active flag).')
0103     parser.add_argument('--activate', type=int, nargs='+', metavar='ID',
0104                         help='set active=Y on these RETRYERRORS ids')
0105     parser.add_argument('--deactivate', type=int, nargs='+', metavar='ID',
0106                         help='set active=N on these RETRYERRORS ids')
0107     parser.add_argument('--apply', action='store_true',
0108                         help='write the switches (dry run without)')
0109     args = parser.parse_args()
0110
0111     if args.activate and args.deactivate:
0112         overlap = set(args.activate) & set(args.deactivate)
0113         if overlap:
0114             print(f'ERROR: ids in both --activate and --deactivate: '
0115                   f'{sorted(overlap)}', file=sys.stderr)
0116             return 1
0117     status = 0
0118     if args.activate:
0119         status = set_rule_active(args.activate, 'Y', args.apply) or status
0120     if args.deactivate:
0121         status = set_rule_active(args.deactivate, 'N', args.apply) or status
0122     if not args.activate and not args.deactivate:
0123         list_rules()
0124         return 0
0125     print()
0126     list_rules()
0127     return status
0128
0129
0130 if __name__ == '__main__':
0131     sys.exit(main())