swf-monitor/scripts/gke-pilot-cycle.py

0001 #!/usr/bin/env python3
0002 """gke-pilot-cycle.py — one cycle of the BNL_ePIC_GOOGLE_es pilot flow:
0003 the queue's activated jobs and our pilot pods in the GKE cluster decide
0004 how many pilot pods to start, and the cycle starts them
0005 (swf-epicprod docs/GKE_PILOT_FLOW.md; logic in swf_epicprod/gke_pilots.py).
0006
0007 The prod-ops agent's doer for ``gke_pilot_cycle``, enqueued by cron.
0008 It starts nothing while SysConfig ``gke_pilots.*`` is incomplete.
0009 Django-bootstrap standalone script, also usable by hand::
0010
0011     cd /data/wenauseic/github/swf-monitor/src
0012     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0013     python ../scripts/gke-pilot-cycle.py [--dry-run] [--created-by NAME]
0014
0015 ``--dry-run`` decides and prints, starting and recording nothing.
0016 Prints one ``SUMMARY`` JSON line; exit 1 on failure.
0017 """
0018 import argparse
0019 import json
0020 import os
0021 import sys
0022
0023 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0024 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0025 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0026
0027 import django  # noqa: E402
0028 django.setup()
0029
0030 from swf_epicprod.gke_pilots import run_cycle  # noqa: E402
0031
0032
0033 def main():
0034     ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
0035     ap.add_argument('--dry-run', action='store_true',
0036                     help='decide and print, start and record nothing')
0037     ap.add_argument('--created-by', default='gke_pilots',
0038                     help='who runs the cycle (the record carries it)')
0039     args = ap.parse_args()
0040     try:
0041         summary = run_cycle(dry_run=args.dry_run, created_by=args.created_by)
0042     except Exception as exc:  # noqa: BLE001
0043         print(f'ERROR: {type(exc).__name__}: {exc}', file=sys.stderr)
0044         return 1
0045     print(f"{summary['queue']}: {summary['outcome']} ({summary['reason']}); "
0046           f"activated {summary['activated']}, started {summary['started']}")
0047     print('SUMMARY ' + json.dumps(summary, default=str))
0048     return 0
0049
0050
0051 if __name__ == '__main__':
0052     sys.exit(main())