swf-monitor/scripts/record-reproduction-fatal.py

0001 #!/usr/bin/env python3
0002 """Record fatal evidence from a preserved reproduction stage log.
0003
0004 For historical jobs predating the fatal watchdog. Default is read-only;
0005 --apply attaches the evidence to the matching ProbeRun without inventing
0006 a payload exit or changing PanDA status. Reconciliation remains separate.
0007 See swf-epicprod/docs/SEGFAULT_DIAGNOSIS.md.
0008 """
0009 import argparse
0010 from datetime import datetime, timezone
0011 import hashlib
0012 import json
0013 import os
0014 from pathlib import Path
0015 import sys
0016
0017 sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
0018 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0019
0020 import django
0021 django.setup()
0022 from django.db import transaction
0023 from canary.store.models import ProbeRun
0024 from swf_epicprod.payload.fatal_watch import fatal_signal, tail
0025
0026
0027 def main():
0028     ap = argparse.ArgumentParser(description=__doc__.split('\n')[0])
0029     ap.add_argument('--job', type=int, required=True)
0030     ap.add_argument('--stage', choices=['simulation', 'reconstruction'], required=True)
0031     ap.add_argument('--log', required=True)
0032     ap.add_argument('--source', required=True, help='original source URL of this job stage log')
0033     ap.add_argument('--apply', action='store_true')
0034     args = ap.parse_args()
0035     text = tail(args.log)
0036     evidence = fatal_signal(text)
0037     if evidence is None:
0038         ap.error('no explicit fatal signal in the stage log tail')
0039     evidence.update(stage=args.stage, source=args.source, log=Path(args.log).name,
0040                     tail=text[-16000:], sha256=hashlib.sha256(Path(args.log).read_bytes()).hexdigest(),
0041                     observed_at=datetime.now(timezone.utc).isoformat())
0042     with transaction.atomic():
0043         runs = ProbeRun.objects.filter(data__kind='payload', data__pandaid=args.job)
0044         if args.apply:
0045             runs = runs.select_for_update()
0046         run = runs.get()
0047         if not (run.data or {}).get('signature'):
0048             ap.error('job is not a signature reproduction')
0049         if args.apply:
0050             run.data = dict(run.data or {}, fatal=evidence)
0051             run.save(update_fields=['data', 'modified_at'])
0052         print(json.dumps({'applied': args.apply, 'job': args.job, 'run': str(run.id),
0053                           'signal': evidence['signal_name'], 'stage': args.stage,
0054                           'source': args.source}))
0055
0056
0057 if __name__ == '__main__':
0058     main()