management/commands/repair_stuck_run_states.py

0001 """Terminalize RunState rows abandoned by their writer — one-time repair.
0002
0003 Two strata of stale state, both repaired here. Rows whose Run record
0004 carries an end_time — the pre-transition-processor era, when RunState
0005 kept its launch state forever — become ended/completed at the run's
0006 true end. Rows whose run was announced but never started or ended (no
0007 Run end_time, non-terminal state, stale beyond --hours) — the residue
0008 of crashed launchers — become abandoned/failed at their last
0009 transition. The runner now terminalizes on every exit path, and the
0010 stale-state System check names any future survivor; this command is
0011 repair, never a schedule. Dry run by default; --apply writes.
0012 """
0013
0014 from datetime import timedelta
0015
0016 from django.core.management.base import BaseCommand
0017 from django.utils import timezone
0018
0019 from monitor_app.models import Run, RunState
0020
0021
0022 TERMINAL_STATES = ('ended', 'expired', 'abandoned')
0023
0024
0025 class Command(BaseCommand):
0026     help = ("Mark RunState rows ended when their Run has an end_time. "
0027             "Dry run unless --apply is given.")
0028
0029     def add_arguments(self, parser):
0030         parser.add_argument(
0031             '--apply', action='store_true',
0032             help='Write the repairs (default is a dry-run listing).')
0033         parser.add_argument(
0034             '--hours', type=float, default=12,
0035             help='staleness threshold for never-ended runs')
0036
0037     def handle(self, *args, **options):
0038         ended_runs = dict(
0039             Run.objects.filter(end_time__isnull=False)
0040             .values_list('run_number', 'end_time'))
0041         cutoff = timezone.now() - timedelta(hours=options['hours'])
0042         candidates = []
0043         abandoned = []
0044         for row in RunState.objects.exclude(
0045                 state__in=TERMINAL_STATES).order_by('run_number'):
0046             if row.run_number in ended_runs:
0047                 candidates.append(row)
0048             elif row.state_changed_at < cutoff:
0049                 abandoned.append(row)
0050         if not candidates and not abandoned:
0051             self.stdout.write('No stuck RunState rows found.')
0052             return
0053
0054         for row in candidates:
0055             self.stdout.write(
0056                 f"run {row.run_number}: {row.state}/{row.substate or '-'} "
0057                 f"({row.phase}) since {row.state_changed_at:%Y-%m-%d %H:%M} "
0058                 f"— run ended {ended_runs[row.run_number]:%Y-%m-%d %H:%M}")
0059         for row in abandoned:
0060             self.stdout.write(
0061                 f"run {row.run_number}: {row.state}/{row.substate or '-'} "
0062                 f"({row.phase}) since {row.state_changed_at:%Y-%m-%d %H:%M} "
0063                 f"— announced, never ended -> abandoned")
0064         self.stdout.write(
0065             f'{len(candidates)} ended-run row(s), '
0066             f'{len(abandoned)} never-ended row(s).')
0067
0068         if not options['apply']:
0069             self.stdout.write('Dry run — nothing written. '
0070                               'Re-run with --apply to repair.')
0071             return
0072
0073         for row in candidates:
0074             row.state = 'ended'
0075             row.substate = None
0076             row.phase = 'completed'
0077             row.state_changed_at = ended_runs[row.run_number]
0078             row.save(update_fields=[
0079                 'state', 'substate', 'phase', 'state_changed_at',
0080                 'updated_at'])
0081         for row in abandoned:
0082             row.state = 'abandoned'
0083             row.substate = None
0084             row.phase = 'failed'
0085             row.save(update_fields=[
0086                 'state', 'substate', 'phase', 'updated_at'])
0087         self.stdout.write(self.style.SUCCESS(
0088             f'Repaired {len(candidates) + len(abandoned)} '
0089             'RunState row(s).'))
0090
0091         # The lanes read the published datataking component, not RunState
0092         # directly — a repair is not done until the surface reflects it.
0093         from monitor_app.snapper_datataking import publish_datataking_state
0094         publish_datataking_state()
0095         self.stdout.write('Datataking component republished.')