swf-monitor/scripts/report-sweep.py

0001 #!/usr/bin/env python3
0002 """report-sweep.py — sweep failed jobs' payload reports out of the store.
0003
0004 The prod-ops agent's doer for the payload report sweep: a bounded sample
0005 per failure signature read and filed beside the job record, the rest of
0006 that signature deleted unread, and a pass record posted to the gateway
0007 whatever the outcome (swf-epicprod docs/JOB_REPORTING.md).
0008
0009 Django-bootstrap standalone script — also usable by hand. Usage::
0010
0011     cd /data/wenauseic/github/swf-monitor/src
0012     ../../swf-testbed/.venv/bin/python ../scripts/report-sweep.py \
0013         [--hours 6] [--limit N] [--per-signature 2] [--dry-run]
0014
0015 --dry-run reads and reports but files nothing, deletes nothing and posts
0016 no pass record, so it can be run against production to see what a pass
0017 would take.
0018 """
0019 import argparse
0020 import json
0021 import os
0022 import sys
0023 from datetime import datetime, timedelta, timezone
0024
0025 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0026 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0027 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0028
0029 import django  # noqa: E402
0030 django.setup()
0031
0032 from monitor_app import payload_reports  # noqa: E402
0033
0034
0035 def main():
0036     parser = argparse.ArgumentParser()
0037     parser.add_argument('--hours', type=float,
0038                         default=payload_reports.DEFAULT_HOURS,
0039                         help='window of failed jobs to consider')
0040     parser.add_argument('--limit', type=int, default=None,
0041                         help='cap the candidates considered, for a first run')
0042     parser.add_argument('--per-signature', type=int,
0043                         default=payload_reports.READ_PER_SIGNATURE,
0044                         help='reports read per distinct failure signature')
0045     parser.add_argument('--dry-run', action='store_true',
0046                         help='read only: file nothing, delete nothing, post nothing')
0047     parser.add_argument('--json', action='store_true',
0048                         help='print the pass record as JSON')
0049     args = parser.parse_args()
0050
0051     since = datetime.now(timezone.utc) - timedelta(hours=args.hours)
0052     print(f'window: since {since.isoformat()}')
0053     record = payload_reports.sweep(
0054         since, limit=args.limit, per_signature=args.per_signature,
0055         dry_run=args.dry_run)
0056     if args.json:
0057         print(json.dumps(record, indent=2, default=str))
0058     print(f"outcome={record['outcome']} candidates={record.get('candidates', 0)} "
0059           f"signatures={record.get('signatures', 0)} filed={len(record['filed'])} "
0060           f"deleted_read={len(record['deleted_read'])} "
0061           f"deleted_unread={len(record['deleted_unread'])} "
0062           f"delivered={record.get('delivered')}")
0063     if record.get('reason'):
0064         print(f"reason: {record['reason']}")
0065     # A finished trial's measurement lives in PanDA's metatable, which the
0066     # sweep above never reads (it files failed jobs' reports); the trial
0067     # cost pass records it on the edition the trial proves (PCS.md, Trials).
0068     if not args.dry_run:
0069         try:
0070             from pcs.services import record_trial_costs_pending
0071             costs = record_trial_costs_pending()
0072             print(f"trial cost: examined={costs['examined']} "
0073                   f"recorded={costs['recorded']} "
0074                   f"unmeasured={len(costs['unmeasured'])}")
0075         except Exception as e:                                # noqa: BLE001
0076             print(f'WARNING: trial cost pass failed: {e}', file=sys.stderr)
0077     return 0 if record['outcome'] != 'failed' else 1
0078
0079
0080 if __name__ == '__main__':
0081     sys.exit(main())