swf-monitor/scripts/import_default_datasets.py

0001 #!/usr/bin/env python3
0002 """
0003 import_default_datasets.py — idempotent import of Sakib's default-datasets CSV
0004 into the PCS Production Task Catalog.
0005
0006 Source: ``eic/epic-prod/docs/_data/datasets.csv``. Each CSV row becomes
0007 one Dataset and one ProdTask (status=``csv_import``) linked to the
0008 current Campaign. Re-running updates rows in place (idempotency key is
0009 ``(Dataset Path, Generator/Dataset Version)``).
0010
0011 The same logic backs the "Update from CSV" button on the catalog page;
0012 this script exists so an operator can run the import from the shell
0013 (or a cron job) without going through the web UI.
0014
0015 Usage::
0016
0017     cd /data/wenauseic/github/swf-monitor/src
0018     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0019     python ../scripts/import_default_datasets.py                 # default path
0020     python ../scripts/import_default_datasets.py path/to/other.csv
0021 """
0022 import os
0023 import sys
0024
0025 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0026 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0027 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0028
0029 import django  # noqa: E402
0030 django.setup()
0031
0032 from pcs.services import (  # noqa: E402
0033     import_default_datasets_csv,
0034     DEFAULT_DATASETS_CSV_PATH,
0035     ServiceError,
0036 )
0037
0038
0039 def main(argv):
0040     csv_path = argv[1] if len(argv) > 1 else DEFAULT_DATASETS_CSV_PATH
0041     print(f'Importing from: {csv_path}')
0042     try:
0043         summary = import_default_datasets_csv(csv_path)
0044     except (ServiceError, FileNotFoundError, OSError) as e:
0045         print(f'ERROR: {e}', file=sys.stderr)
0046         return 1
0047     print(f'  rows:    {summary["rows"]}')
0048     print(f'  created: {summary["created"]}')
0049     print(f'  updated: {summary["updated"]}')
0050     if summary['errors']:
0051         print(f'  errors:  {len(summary["errors"])}')
0052         for err in summary['errors'][:10]:
0053             print(f'    - {err}')
0054         if len(summary['errors']) > 10:
0055             print(f'    ... and {len(summary["errors"]) - 10} more')
0056     return 0
0057
0058
0059 if __name__ == '__main__':
0060     sys.exit(main(sys.argv))