swf-monitor/scripts/worker-record-capture.py

0001 #!/usr/bin/env python3
0002 """worker-record-capture.py — copy the harvester worker records of the
0003 allocation page's queues before PanDA drops them.
0004
0005 The prod-ops agent's doer for ``worker_record_capture``, nightly by cron
0006 enqueue: every worker row at the NERSC queues updated in the window is
0007 copied into our store (monitor_app.worker_records), so an allocation keeps
0008 its batch start, end and cores after PanDA's three-month window
0009 (swf-epicprod docs/EPICPROD_OPS.md, Harvester worker records).
0010
0011 Django-bootstrap standalone script, also usable by hand::
0012
0013     cd /data/wenauseic/github/swf-monitor/src
0014     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0015     python ../scripts/worker-record-capture.py [--days 100]
0016 """
0017 import argparse
0018 import os
0019 import sys
0020
0021 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0022 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0023 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0024
0025 import django  # noqa: E402
0026 django.setup()
0027
0028 from monitor_app import worker_records  # noqa: E402
0029
0030
0031 def main():
0032     parser = argparse.ArgumentParser()
0033     parser.add_argument('--days', type=float, default=worker_records.DEFAULT_DAYS,
0034                         help='window of worker updates to copy (default 100, all PanDA holds)')
0035     args = parser.parse_args()
0036     counts = worker_records.capture(days=args.days)
0037     print(f"read={counts['read']} created={counts['created']} updated={counts['updated']} "
0038           f"unchanged={counts['unchanged']}")
0039     return 0
0040
0041
0042 if __name__ == '__main__':
0043     sys.exit(main())