swf-monitor/scripts/pcs-catalog-import.py

0001 #!/usr/bin/env python3
0002 """pcs-catalog-import.py — run a PCS catalog import in the background.
0003
0004 The prod-ops agent's doer for the catalog 'Update from CSV' and 'Update from
0005 epic-prod' buttons: the web view publishes ``catalog_import`` to the agent,
0006 which runs this script off the WSGI request (the epic-prod walk of ~4900
0007 datasets times the gateway out) and pushes ``catalog_import_ready`` to the
0008 browser when done. Django-bootstrap standalone — also usable by hand or cron.
0009 See ``docs/EPICPROD_OPS_AGENT.md`` and ``docs/SSE_PUSH.md``.
0010
0011 Usage::
0012
0013     cd /data/wenauseic/github/swf-monitor/src
0014     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0015     python ../scripts/pcs-catalog-import.py {csv|epic-prod}
0016 """
0017 import os
0018 import sys
0019
0020 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0021 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0022 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0023
0024 import django  # noqa: E402
0025 django.setup()
0026
0027
0028 def main(argv):
0029     source = argv[1] if len(argv) > 1 else ''
0030     from pcs.services import (import_default_datasets_csv,
0031                               import_epic_prod_past_campaigns, ServiceError)
0032     try:
0033         if source == 'csv':
0034             s = import_default_datasets_csv(created_by='prodops_agent')
0035             print(f"csv: {s['created']} new, {s['updated']} updated, "
0036                   f"{len(s['errors'])} errors (of {s['rows']} rows)")
0037         elif source == 'epic-prod':
0038             s = import_epic_prod_past_campaigns(created_by='prodops_agent')
0039             print(f"epic-prod: {s['created']} new, {s['updated']} updated, across "
0040                   f"{s['campaigns']} campaigns, {len(s['errors'])} errors "
0041                   f"(of {s['rows']} rows)")
0042         else:
0043             print(f"unknown source {source!r}; expected 'csv' or 'epic-prod'",
0044                   file=sys.stderr)
0045             return 2
0046     except (ServiceError, FileNotFoundError, OSError) as e:
0047         print(f"ERROR: {e}", file=sys.stderr)
0048         return 1
0049     # Per-row warnings are not failures — surface them, but the import succeeded.
0050     for err in s['errors']:
0051         print(f"  warn: {err}", file=sys.stderr)
0052     return 0
0053
0054
0055 if __name__ == '__main__':
0056     sys.exit(main(sys.argv))