swf-monitor/scripts/batch-log-capture.py

0001 #!/usr/bin/env python3
0002 """batch-log-capture.py — capture condor event logs while they exist.
0003
0004 The prod-ops agent's doer for the batch-record capture: every failed
0005 job's condor event log fetched whole from the harvester and written to
0006 the file store, and date directories past the retention window removed.
0007 The source keeps about eighteen days, so a log not captured is a reason
0008 that cannot be recovered or re-parsed later
0009 (docs/ERROR_ATTRIBUTION.md, Batch-layer records).
0010
0011 Django-bootstrap standalone script — also usable by hand. Usage::
0012
0013     cd /data/wenauseic/github/swf-monitor/src
0014     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0015     python ../scripts/batch-log-capture.py [--hours 24] [--limit N]
0016                                            [--prune] [--dry-run]
0017 """
0018 import argparse
0019 import os
0020 import sys
0021 from datetime import datetime, timedelta, timezone
0022
0023 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0024 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0025 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0026
0027 import django  # noqa: E402
0028 django.setup()
0029
0030 from monitor_app import batch_records  # noqa: E402
0031
0032
0033 def main():
0034     parser = argparse.ArgumentParser()
0035     parser.add_argument('--hours', type=float, default=24,
0036                         help='window of failed jobs to consider (default 24)')
0037     parser.add_argument('--limit', type=int, default=None,
0038                         help='cap the candidates considered, for a first run')
0039     parser.add_argument('--prune', action='store_true',
0040                         help='also remove date directories past retention')
0041     parser.add_argument('--dry-run', action='store_true',
0042                         help='with --prune, name the directories only')
0043     parser.add_argument('--root', default=None,
0044                         help='override the store root')
0045     args = parser.parse_args()
0046
0047     root = args.root or batch_records.store_root()
0048     since = datetime.now(timezone.utc) - timedelta(hours=args.hours)
0049     print(f'store: {root}')
0050     print(f'window: since {since.isoformat()}')
0051
0052     counts = batch_records.capture(since=since, limit=args.limit, root=root)
0053     print(f"candidates={counts['candidates']} captured={counts['captured']} "
0054           f"failed={counts['failed']} skipped={counts['skipped']}")
0055
0056     if args.prune:
0057         removed = batch_records.prune(root=root, dry_run=args.dry_run)
0058         verb = 'would remove' if args.dry_run else 'removed'
0059         print(f'{verb} {len(removed)} date directory(ies)')
0060         for path in removed:
0061             print(f'  {path}')
0062     return 0
0063
0064
0065 if __name__ == '__main__':
0066     sys.exit(main())