swf-monitor/scripts/harvester-stdout-capture.py

0001 #!/usr/bin/env python3
0002 """harvester-stdout-capture.py — copy the harvester's stdout of the jobs
0003 worth keeping while the PanDA server still holds it.
0004
0005 The prod-ops agent's doer for ``harvester_stdout_capture``, hourly by
0006 cron enqueue: every failed job (and finished, when the switch is on)
0007 whose pilot id names a PanDA-cache stdout has that stdout copied whole
0008 into our store, where the server's seven-day cache purge cannot reach
0009 it; copies past ``harvester_stdout.keep_days`` are pruned (0 keeps
0010 them) (docs/EPICPROD_OPS.md, Harvester stdout records).
0011
0012 Django-bootstrap standalone script, also usable by hand::
0013
0014     cd /data/wenauseic/github/swf-monitor/src
0015     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0016     python ../scripts/harvester-stdout-capture.py [--days 7] [--limit N]
0017                                                   [--finished] [--prune] [--dry-run]
0018 """
0019 import argparse
0020 import os
0021 import sys
0022 from datetime import datetime, timedelta, timezone
0023
0024 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0025 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0026 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0027
0028 import django  # noqa: E402
0029 django.setup()
0030
0031 from monitor_app import harvester_stdout  # noqa: E402
0032
0033
0034 def main():
0035     parser = argparse.ArgumentParser()
0036     parser.add_argument('--days', type=float, default=None,
0037                         help='window of ended jobs to consider (default: the backfill_days setting, 7)')
0038     parser.add_argument('--limit', type=int, default=None, help='cap the candidates considered')
0039     parser.add_argument('--finished', action='store_true',
0040                         help='include finished jobs whatever the switch says')
0041     parser.add_argument('--prune', action='store_true', help='also prune copies past keep_days')
0042     parser.add_argument('--dry-run', action='store_true', help='with --prune, name the files only')
0043     parser.add_argument('--root', default=None, help='override the store root')
0044     args = parser.parse_args()
0045
0046     root = args.root or harvester_stdout.store_root()
0047     since = (datetime.now(timezone.utc) - timedelta(days=args.days)) if args.days else None
0048     print(f'store: {root}')
0049     counts, captured, gone = harvester_stdout.capture(
0050         since=since, finished=True if args.finished else None, limit=args.limit, root=root)
0051     print(f"candidates={counts['candidates']} captured={counts['captured']} gone={counts['gone']} "
0052           f"failed={counts['failed']} skipped={counts['skipped']} bytes={counts['bytes']}")
0053     if captured:
0054         print('captured: ' + ', '.join(str(p) for p in captured[:50]))
0055     if gone:
0056         print('gone from the cache before capture: ' + ', '.join(str(p) for p in gone[:50]))
0057     if args.prune:
0058         removed = harvester_stdout.prune(root=root, dry_run=args.dry_run)
0059         print(f"{'would remove' if args.dry_run else 'removed'} {len(removed)} file(s)")
0060     return 0
0061
0062
0063 if __name__ == '__main__':
0064     sys.exit(main())