swf-monitor/scripts/rucio-arrivals-sweep.py

0001 #!/usr/bin/env python3
0002 """rucio-arrivals-sweep.py — detect new files landing in JLab Rucio.
0003
0004 The prod-ops agent's doer for the clockwork arrivals sweep (a step of the
0005 nightly ``catalog_sync`` chain): one ``created_after`` DID query per root
0006 across all campaign versions, per-campaign arrivals recorded on the
0007 Campaign rows, one live ``rucio_arrivals`` event when anything arrived.
0008 Django-bootstrap standalone script — also usable by hand. See
0009 ``docs/EPICPROD_DATA_LINEAGE.md``.
0010
0011 Usage::
0012
0013     cd /data/wenauseic/github/swf-monitor/src
0014     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0015     python ../scripts/rucio-arrivals-sweep.py [--window-hours 24]
0016 """
0017 import argparse
0018 import os
0019 import sys
0020
0021 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0022 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0023 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0024
0025 import django  # noqa: E402
0026 django.setup()
0027
0028 from pcs.services import sweep_rucio_arrivals  # noqa: E402
0029
0030
0031 def main():
0032     parser = argparse.ArgumentParser()
0033     parser.add_argument('--window-hours', type=float, default=None,
0034                         help='override the since-last-sweep window')
0035     parser.add_argument('--created-by', default='prodops_agent')
0036     args = parser.parse_args()
0037
0038     result = sweep_rucio_arrivals(
0039         window_hours=args.window_hours,
0040         created_by=args.created_by,
0041         instance='catalog-sync',
0042     )
0043     print(f"total_files={result['total_files']} "
0044           f"window_start={result['window_start']}")
0045     for name, files in sorted(result['campaigns'].items()):
0046         marker = '' if name in result['known'] else '  [no catalog campaign]'
0047         print(f'  {name}: {files} file(s){marker}')
0048     return 0
0049
0050
0051 if __name__ == '__main__':
0052     sys.exit(main())