swf-monitor/scripts/measure-file-events.py

0001 #!/usr/bin/env python3
0002 """measure-file-events.py — per-file event measurement for delivered
0003 campaign data.
0004
0005 The prod-ops agent's doer for the nightly measurement pass (a step of
0006 the ``catalog_sync`` chain): anchor each new byte-size class with one
0007 xrootd read on a disk replica — never tape — and fill class members at
0008 the anchored rate; tape-only classes of dormant complete sources derive
0009 from the ANL catalog. Incremental: already-measured files are skipped,
0010 so the nightly pass costs a few file opens at most. Logic in
0011 ``swf_epicprod/analytics/file_events.py``; see
0012 ``swf-epicprod/docs/CAMPAIGN_DELIVERY.md`` (The events source).
0013 Django-bootstrap standalone script — also usable by hand.
0014
0015 Usage::
0016
0017     cd /data/wenauseic/github/swf-monitor/src
0018     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0019     python ../scripts/measure-file-events.py [--workers 6]
0020 """
0021 import argparse
0022 import json
0023 import os
0024 import sys
0025
0026 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0027 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0028 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0029
0030 import django  # noqa: E402
0031 django.setup()
0032
0033 from swf_epicprod.analytics.file_events import measure_file_events  # noqa: E402
0034
0035
0036 def main():
0037     parser = argparse.ArgumentParser()
0038     parser.add_argument('--campaigns', default='',
0039                         help='comma-separated campaign families '
0040                              '(default: all in the catalog)')
0041     parser.add_argument('--workers', type=int, default=6)
0042     parser.add_argument('--locations', type=int, default=0,
0043                         help='process at most N locations (0 = all)')
0044     args = parser.parse_args()
0045     campaigns = [c.strip() for c in args.campaigns.split(',')
0046                  if c.strip()] or None
0047
0048     stats = measure_file_events(campaigns, workers=args.workers,
0049                                 max_locations=args.locations)
0050     print('SUMMARY ' + json.dumps(stats))
0051     return 0
0052
0053
0054 if __name__ == '__main__':
0055     sys.exit(main())