swf-monitor/scripts/es-closeout-cycle.py

0001 #!/usr/bin/env python3
0002 """es-closeout-cycle.py — one cycle of the preemption close-out: a running
0003 Event Service job at a queue listed in SysConfig ``es_closeout.queues``
0004 whose shipped record has gone quiet is credited and finished (swf-epicprod
0005 docs/NODE_EVENT_DISPATCHER.md, Preemption; logic in
0006 swf_epicprod/es_closeout.py).
0007
0008 The prod-ops agent's doer for ``es_closeout_cycle``, every five minutes by
0009 cron enqueue. Django-bootstrap standalone script, also usable by hand::
0010
0011     cd /data/wenauseic/github/swf-monitor/src
0012     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0013     python ../scripts/es-closeout-cycle.py [--dry-run] [--created-by NAME]
0014
0015 ``--dry-run`` judges and prints, and closes and records nothing.
0016 Prints one ``SUMMARY`` JSON line; exit 1 on failure.
0017 """
0018 import argparse
0019 import json
0020 import os
0021 import sys
0022
0023 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0024 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0025 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0026
0027 import django  # noqa: E402
0028 django.setup()
0029
0030 from swf_epicprod.es_closeout import run_cycle  # noqa: E402
0031
0032
0033 def main():
0034     ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
0035     ap.add_argument('--dry-run', action='store_true',
0036                     help='judge and print, close and record nothing')
0037     ap.add_argument('--created-by', default='es-closeout',
0038                     help='who runs the cycle (the records carry it)')
0039     ap.add_argument('--records-only', action='store_true',
0040                     help='refresh running harness snapshots without closing jobs')
0041     args = ap.parse_args()
0042     try:
0043         from monitor_app.payload_reports import refresh_running_es_records
0044         if not args.dry_run:
0045             try:
0046                 filed = refresh_running_es_records()
0047                 print('RECORDS ' + json.dumps({'filed': filed}))
0048             except Exception as exc:  # noqa: BLE001
0049                 print(f'ERROR: snapshot refresh: {type(exc).__name__}: {exc}', file=sys.stderr)
0050                 if args.records_only:
0051                     return 1
0052         if args.records_only:
0053             return 0
0054         decisions, summary = run_cycle(dry_run=args.dry_run, created_by=args.created_by)
0055     except Exception as exc:  # noqa: BLE001
0056         print(f'ERROR: {type(exc).__name__}: {exc}', file=sys.stderr)
0057         return 1
0058     for d in decisions:
0059         r = d.get('result') or {}
0060         outcome = ('' if 'result' not in d else
0061                    ' closed' if r.get('ok') else f" NOT closed: {r.get('refused') or r.get('error') or r.get('update')}")
0062         print(f"{d['pandaid']} {d['queue']}: {'quiet' if d.get('act') else 'live'} ({d['reason']}){outcome}")
0063     print('SUMMARY ' + json.dumps(summary, default=str))
0064     return 1 if summary.get('failed') else 0
0065
0066
0067 if __name__ == '__main__':
0068     sys.exit(main())