swf-monitor/scripts/rse-deletion.py

0001 #!/usr/bin/env python3
0002 """rse-deletion.py — the prod-ops agent's doer for the ``rse_deletion``
0003 message: preview or remove datasets matching a pattern on a disk RSE,
0004 only where tape holds a complete copy and the producing task is not
0005 active.
0006
0007 Logic in ``swf_epicprod/rse_deletion.py``; design in swf-epicprod
0008 ``docs/STORAGE.md`` (Removing datasets from an RSE). The catalog is JLab
0009 Rucio as ``eicprod`` by a PanDA-server token, else the proxy in
0010 EVGEN_X509_PROXY; task state is the storage pass's reading of PCS and
0011 PanDA. Django-bootstrap standalone script.
0012
0013 Prints JSON lines on stdout, each flushed as written: for a removal, one
0014 ``{"event": "outcome", ...}`` line per dataset as it is decided, so a
0015 removal cut short leaves what it did; then one ``{"event": "result",
0016 ...}`` line with the plan (preview) or the outcomes and totals
0017 (execute), or ``{"event": "error", "error": ...}``. Exit 0 on success, 1
0018 for a refused request or unreachable catalog, 2 for a removal with
0019 failures.
0020
0021 Usage::
0022
0023     cd /data/wenauseic/github/swf-monitor/src
0024     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0025     python ../scripts/rse-deletion.py --phase preview --rse BNL-XRD --pattern '/RECO/26.03.0/*'
0026     python ../scripts/rse-deletion.py --phase execute --rse BNL-XRD --pattern '/RECO/26.03.0/*' \\
0027         --names-file names.json
0028 """
0029 import argparse
0030 import json
0031 import os
0032 import sys
0033
0034 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0035 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0036 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0037
0038 import django  # noqa: E402
0039 django.setup()
0040
0041 from swf_epicprod import rse_deletion  # noqa: E402
0042 from swf_epicprod.analytics.storage import _task_states  # noqa: E402
0043 from swf_epicprod.jlab_auth import rucio_client  # noqa: E402
0044
0045
0046 def emit(event, **body):
0047     print(json.dumps(dict(body, event=event)), flush=True)
0048
0049
0050 def main():
0051     parser = argparse.ArgumentParser()
0052     parser.add_argument('--phase', choices=('preview', 'execute'), required=True)
0053     parser.add_argument('--rse', required=True)
0054     parser.add_argument('--pattern', required=True)
0055     parser.add_argument('--names-file', default='',
0056                         help='execute: JSON list of the dataset names to remove')
0057     args = parser.parse_args()
0058
0059     try:
0060         rse_deletion.validate(args.rse, args.pattern)
0061         client = rucio_client(proxy=os.environ.get('EVGEN_X509_PROXY'), timeout=60)
0062         auth = getattr(client, 'jlab_auth', '')
0063         if args.phase == 'preview':
0064             out = rse_deletion.plan(client, args.rse, args.pattern, _task_states)
0065             emit('result', auth=auth, **out)
0066             return 0
0067         with open(args.names_file) as f:
0068             names = json.load(f)
0069         out = rse_deletion.execute(
0070             client, args.rse, args.pattern, names, _task_states,
0071             on_outcome=lambda o: emit('outcome', **o))
0072         emit('result', auth=auth, **out)
0073         return 2 if out['totals']['failed'] else 0
0074     except rse_deletion.RseDeletionError as e:
0075         emit('error', error=str(e))
0076     except Exception as e:                                    # noqa: BLE001
0077         emit('error', error=f'{type(e).__name__}: {e}')
0078     return 1
0079
0080
0081 if __name__ == '__main__':
0082     sys.exit(main())