swf-monitor/scripts/node-guard-cycle.py

0001 #!/usr/bin/env python3
0002 """node-guard-cycle.py — one cycle of the node guard: the terminal
0003 production jobs of the window per queue and host go to the canary's
0004 decision, and the verdicts are recorded on the action stream and stored
0005 for the Node guard page (site-canary docs/NODE_GUARD.md; logic in
0006 monitor_app/panda/node_guard.py).
0007
0008 The prod-ops agent's doer for ``node_guard_cycle``, every five minutes
0009 by cron enqueue. In shadow mode (SysConfig ``node_guard.mode``) it acts
0010 on nothing and records what it would have done. Django-bootstrap
0011 standalone script, also usable by hand::
0012
0013     cd /data/wenauseic/github/swf-monitor/src
0014     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0015     python ../scripts/node-guard-cycle.py [--dry-run] [--created-by NAME]
0016
0017 ``--dry-run`` judges and prints, and writes no record.
0018 Prints one ``SUMMARY`` JSON line; exit 1 on failure.
0019 """
0020 import argparse
0021 import json
0022 import os
0023 import sys
0024
0025 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0026 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0027 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0028
0029 import django  # noqa: E402
0030 django.setup()
0031
0032 from monitor_app.panda.node_guard import run_cycle  # noqa: E402
0033
0034
0035 def main():
0036     ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
0037     ap.add_argument('--dry-run', action='store_true',
0038                     help='judge and print, record nothing')
0039     ap.add_argument('--created-by', default='node-guard',
0040                     help='who runs the cycle (the records carry it)')
0041     args = ap.parse_args()
0042     try:
0043         nodes, summary = run_cycle(dry_run=args.dry_run, created_by=args.created_by)
0044     except Exception as exc:  # noqa: BLE001
0045         print(f'ERROR: {type(exc).__name__}: {exc}', file=sys.stderr)
0046         return 1
0047     for n in nodes:
0048         e = n['evidence']
0049         print(f"{n['queue']} {n['host']}: {n['state']} ({n['reason']}) "
0050               f"{e['failed']}/{e['jobs']} failed, {e['fast_failed']} fast, "
0051               f"{len(e['tasks_finished_elsewhere'])}/{len(e['tasks_failed'])} tasks finish elsewhere"
0052               f"{' recorded' if n['recorded'] and not args.dry_run else ''}")
0053     print('SUMMARY ' + json.dumps(summary, default=str))
0054     return 0
0055
0056
0057 if __name__ == '__main__':
0058     sys.exit(main())