swf-monitor/scripts/storage-door-cycle.py

0001 #!/usr/bin/env python3
0002 """storage-door-cycle.py — one cycle of the storage door canary: the
0003 doors production writes through, from the catalog, used rather than
0004 inspected — a small write, a stat, a delete, and the date on the
0005 certificate they serve — and the verdicts recorded for the page and for
0006 production operations (site-canary docs/STORAGE_DOORS.md; logic in
0007 monitor_app/panda/storage_doors.py).
0008
0009 The prod-ops agent's doer for ``storage_door_cycle``, every fifteen
0010 minutes by cron enqueue; a door is probed when its last probe is older
0011 than ``storage_doors.interval_h`` (one hour by default), so the cadence
0012 is a setting. Nothing acts on the record automatically: a door that
0013 changes verdict raises its action, at alarm severity when it goes down,
0014 and production operations decide what follows.
0015
0016 Django-bootstrap standalone script, also usable by hand::
0017
0018     cd /data/wenauseic/github/swf-monitor/src
0019     source ../../swf-testbed/.venv/bin/activate && source ~/.env
0020     python ../scripts/storage-door-cycle.py [--dry-run] [--force]
0021
0022 ``--dry-run`` probes and prints, and writes no record. ``--force``
0023 probes every door whatever the interval and whatever the switch says,
0024 for a look by hand; it still records unless ``--dry-run`` says not to.
0025 Prints one ``SUMMARY`` JSON line; exit 1 on failure.
0026 """
0027 import argparse
0028 import json
0029 import os
0030 import sys
0031
0032 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0033 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0034 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0035
0036 import django  # noqa: E402
0037 django.setup()
0038
0039 # The probe writes as production writes: at a door that takes no token,
0040 # with the registrar's proxy (EVGEN_X509_PROXY), which the cycle hands to
0041 # the door's command alone (storage_doors -> jlab_auth.door_access). On
0042 # 2026-09-22 the cycle used longproxy-for-rucio, which is group-readable,
0043 # and xrootd refuses a credential with excessive access rights — the write
0044 # fell back to no credential and BNL-XRD answered "permission denied",
0045 # which read as a door that was down while it was taking production's
0046 # bytes. X509_USER_PROXY stays the robot proxy: it signs in to the PanDA
0047 # server for the tokens, which the eicprod proxy cannot do (when the cycle
0048 # put that proxy there and it expired on 2026-10-28, no cycle got a token).
0049
0050 from monitor_app.panda.storage_doors import PROGRESS_PREFIX, run_cycle  # noqa: E402
0051
0052
0053 def main():
0054     ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
0055     ap.add_argument('--dry-run', action='store_true',
0056                     help='probe and print, record nothing')
0057     ap.add_argument('--force', action='store_true',
0058                     help='probe every door, whatever the interval or the switch')
0059     ap.add_argument('--rse', action='append', default=[],
0060                     help='probe only these RSEs (repeatable), for a look by hand')
0061     ap.add_argument('--created-by', default='storage-doors',
0062                     help='who runs the cycle (the records carry it)')
0063     args = ap.parse_args()
0064     try:
0065         doors, summary = run_cycle(dry_run=args.dry_run, force=args.force,
0066                                    only=tuple(args.rse), created_by=args.created_by,
0067                                    progress=lambda event: print(
0068                                        PROGRESS_PREFIX + json.dumps(event), flush=True))
0069     except Exception as exc:  # noqa: BLE001
0070         print(f'ERROR: {type(exc).__name__}: {exc}', file=sys.stderr)
0071         return 1
0072     for rse, state in sorted(doors.items()):
0073         evidence = state.get('evidence') or {}
0074         certificate = (f", certificate {evidence['certificate']} "
0075                        f"({evidence['certificate_days_left']} days)"
0076                        if evidence.get('certificate') else '')
0077         print(f"{rse:12} {state.get('verdict', '-'):8} "
0078               f"{state.get('reason', ''):20} {state.get('door', '')}{certificate}")
0079     print('SUMMARY ' + json.dumps(summary, default=str))
0080     return 0
0081
0082
0083 if __name__ == '__main__':
0084     sys.exit(main())