templates/pcs/pcs_catalog_past.html

0001 {% extends "base.html" %}
0002 {% load swf_fmt %}
0003 {% block title %}Production Task Catalog — {% if active_lifecycle == 'last' %}Last{% else %}Past{% endif %}{% endblock %}
0004
0005 {% block content %}
0006 <style>
0007 .btn-outline-last-green {
0008   color: #5fa380; border-color: #5fa380;
0009 }
0010 .btn-outline-last-green:hover,
0011 .btn-outline-last-green.active {
0012   background-color: #5fa380; color: #fff; border-color: #5fa380;
0013 }
0014
0015 .pcs-pdl-nav a { margin-right: 0.8em; color: #007bff; text-decoration: none; }
0016 .pcs-pdl-nav a:hover { text-decoration: underline; }
0017 .pcs-pdl-nav a.active { font-weight: bold; text-decoration: underline; color: #0056b3; }
0018 .pcs-pdl-section-label { font-weight: 600; margin-right: 0.6em; }
0019 .pcs-pdl-summary { margin-bottom: 0.4em; }
0020 .pcs-pdl-list td, .pcs-pdl-list th { vertical-align: top; }
0021 .pcs-pdl-rse-bad { color: #dc3545; }
0022 .pcs-pdl-filterbar a.filter-link { margin-right: 0.6em; color: #007bff; text-decoration: none; }
0023 .pcs-pdl-filterbar a.filter-link:hover { text-decoration: underline; }
0024 .pcs-pdl-filterbar a.filter-link.filter-active { font-weight: bold; text-decoration: underline; color: #0056b3; }
0025 .pcs-pdl-filterbar a.filter-link.pcs-pdl-incomplete { color: #dc3545; }
0026 .pcs-pdl-filterbar a.filter-link.pcs-pdl-incomplete.filter-active { color: #a71d2a; }
0027 </style>
0028
0029 <div class="container-fluid mt-3 pcs-catalog">
0030   <div class="d-flex align-items-center mb-3">
0031     <h2 class="me-3 mb-0">Production Task Catalog</h2>
0032     <a href="{% url 'pcs:pcs_hub' %}" class="btn btn-sm btn-outline-secondary">PCS Hub</a>
0033     {% if active_lifecycle == 'last' %}
0034     <form method="post" action="{% url 'pcs:pcs_catalog_rucio_update' %}" class="ms-2 d-inline">
0035       {% csrf_token %}
0036       <button type="submit" class="btn btn-sm btn-outline-primary"
0037               title="Refresh the JLab Rucio output snapshot for current + last"
0038               onclick="return confirm('Refresh the JLab Rucio snapshot for current and last campaigns?');">
0039         Update from Rucio
0040       </button>
0041     </form>
0042     {% else %}
0043     <form method="post" action="{% url 'pcs:pcs_catalog_past_update' %}" class="ms-2 d-inline">
0044       {% csrf_token %}
0045       <button type="submit" class="btn btn-sm btn-outline-primary"
0046               title="Re-import 2026 FULL/RECO campaign output from epic-prod"
0047               onclick="return confirm('Re-import 2026 past-campaign output datasets from epic-prod?');">
0048         Update from epic-prod
0049       </button>
0050     </form>
0051     {% endif %}
0052   </div>
0053
0054   {% if show_tabs %}
0055   <div class="pcs-tlf-tabs btn-group mb-2 flex-wrap" role="tablist" aria-label="Lifecycle tabs">
0056     {% for tab in lifecycle_tabs %}
0057       <a class="btn btn-sm btn-outline-{{ tab.color }}{% if active_lifecycle == tab.key %} active{% endif %}"
0058          href="?lifecycle={{ tab.key }}">
0059         {{ tab.label }}{% if tab.key != 'past' and tab.campaigns %} <span class="text-muted">·</span>
0060           {% for c in tab.campaigns %}{{ c.name }}{% if not forloop.last %}, {% endif %}{% endfor %}
0061         {% endif %}
0062       </a>
0063     {% endfor %}
0064   </div>
0065   {% endif %}
0066
0067   {% if active_lifecycle == 'last' and not rucio_unmatched_campaign and rucio_detected %}
0068     <div class="border rounded p-2 mb-3 small bg-light">
0069       <div class="mb-1 text-muted">
0070         No <strong>last</strong> release set. Pick one from the active 26.x releases
0071         in JLab Rucio (PCS current: <strong>{{ rucio_current_name }}</strong>):
0072       </div>
0073       <div class="d-flex flex-wrap align-items-center gap-2">
0074       {% for r in rucio_detected %}
0075         {% if r.version != rucio_current_name %}
0076         <span>{{ r.version }} <span class="text-muted">({{ r.count }})</span>
0077           <form method="post" action="{% url 'pcs:pcs_catalog_set_last' %}" class="d-inline">
0078             {% csrf_token %}
0079             <input type="hidden" name="name" value="{{ r.version }}">
0080             <button type="submit" class="btn btn-sm btn-outline-info py-0 px-2 ms-1"
0081                     title="Designate {{ r.version }} as PCS last and pull its Rucio snapshot"
0082                     onclick="return confirm('Set PCS last to {{ r.version }} and pull snapshot?');">Make last</button>
0083           </form>
0084         </span>
0085         {% endif %}
0086       {% endfor %}
0087       </div>
0088     </div>
0089   {% endif %}
0090
0091   {% if active_lifecycle == 'last' and rucio_timeline_json != 'null' %}
0092     <div id="pcs-rucio-timeline" class="mb-3" style="min-height:540px"></div>
0093   {% endif %}
0094
0095   {% if rucio_unmatched %}
0096   <details class="mb-3 small">
0097     <summary class="text-muted">
0098       <strong style="color:#dc3545">&#9888; {{ rucio_unmatched|length }} unmatched Rucio output{{ rucio_unmatched|length|pluralize }}</strong>
0099       for campaign {{ rucio_unmatched_campaign }} (no past-output row matched these DIDs)
0100     </summary>
0101     <ul class="mt-2 mb-0" style="font-family:monospace; font-size:0.85em;">
0102     {% for u in rucio_unmatched %}
0103       <li>{{ u.did }} &middot; {{ u.files }} files &middot; {{ u.bytes|filesizeformat }}{% if u.incomplete %} <span style="color:#dc3545">(incomplete)</span>{% endif %}</li>
0104     {% endfor %}
0105     </ul>
0106   </details>
0107   {% endif %}
0108
0109   {% if active_lifecycle != 'last' %}
0110   <div class="pcs-pdl-nav small mb-2">
0111     <span class="pcs-pdl-section-label">Release:</span>
0112     <a href="?lifecycle=past&amp;release=all{% if active_stage %}&amp;stage={{ active_stage }}{% endif %}"
0113        class="{% if active_release == 'all' %}active{% endif %}"
0114        style="font-weight:600">All</a>
0115     {% for grp in release_year_groups %}
0116       <span class="text-muted mx-1">|</span>
0117       <a href="?lifecycle=past&amp;release={{ grp.all_key }}{% if active_stage %}&amp;stage={{ active_stage }}{% endif %}"
0118          class="{% if active_release == grp.all_key %}active{% endif %}"
0119          style="font-weight:600">All {{ grp.year }}</a>
0120       {% for v in grp.versions %}
0121         <a href="?lifecycle=past&amp;release={{ v|urlencode }}{% if active_stage %}&amp;stage={{ active_stage }}{% endif %}"
0122            class="{% if active_release == v %}active{% endif %}">{{ v }}</a>
0123       {% endfor %}
0124     {% endfor %}
0125   </div>
0126   {% endif %}
0127
0128   <div class="pcs-pdl-nav small mb-2">
0129     <span class="pcs-pdl-section-label">Stage:</span>
0130     <a href="?lifecycle=past&amp;release={{ active_release|urlencode }}"
0131        class="{% if not active_stage %}active{% endif %}">All ({{ stage_counts.all }})</a>
0132     {% if stage_counts.FULL %}
0133     <a href="?lifecycle=past&amp;release={{ active_release|urlencode }}&amp;stage=FULL"
0134        class="{% if active_stage == 'FULL' %}active{% endif %}">Simu ({{ stage_counts.FULL }})</a>
0135     {% endif %}
0136     {% if stage_counts.RECO %}
0137     <a href="?lifecycle=past&amp;release={{ active_release|urlencode }}&amp;stage=RECO"
0138        class="{% if active_stage == 'RECO' %}active{% endif %}">Reco ({{ stage_counts.RECO }})</a>
0139     {% endif %}
0140   </div>
0141
0142   {% if active_release %}
0143     <div id="pcs-pdl-filterbar" class="pcs-pdl-filterbar small mb-2">
0144       <div class="mb-1">
0145         <a href="#" data-toggle="incomplete" class="filter-link pcs-pdl-incomplete">
0146           Incomplete file counts (<span data-count="incomplete">0</span>)
0147         </a>
0148       </div>
0149       <div class="mb-1 d-flex align-items-center gap-2">
0150         <span class="pcs-pdl-section-label">Search:</span>
0151         <input type="search" id="pcs-pdl-q" class="form-control form-control-sm"
0152                style="width:24em;" placeholder="dataset / path&hellip;">
0153         <button type="button" id="pcs-pdl-clear" class="btn btn-sm btn-outline-secondary"
0154                 title="Clear all filters">Clear</button>
0155       </div>
0156       <div class="mb-1">
0157         <span class="pcs-pdl-section-label">Geometry:</span>
0158         <a href="#" data-filter="detector" data-value="" class="filter-link filter-active">All</a>
0159         <span id="pcs-pdl-detector-opts"></span>
0160       </div>
0161       <div class="mb-1">
0162         <span class="pcs-pdl-section-label">Beam:</span>
0163         <a href="#" data-filter="beam" data-value="" class="filter-link filter-active">All</a>
0164         <span id="pcs-pdl-beam-opts"></span>
0165       </div>
0166       <div class="mb-1">
0167         <span class="pcs-pdl-section-label">Physics:</span>
0168         <a href="#" data-filter="physics" data-value="" class="filter-link filter-active">All</a>
0169         <span id="pcs-pdl-physics-opts"></span>
0170       </div>
0171       <div class="mb-1">
0172         <span class="pcs-pdl-section-label">Q&sup2;:</span>
0173         <a href="#" data-filter="q2" data-value="" class="filter-link filter-active">All</a>
0174         <span id="pcs-pdl-q2-opts"></span>
0175       </div>
0176       <div class="mb-1">
0177         <span class="pcs-pdl-section-label">Species:</span>
0178         <a href="#" data-filter="species" data-value="" class="filter-link filter-active">All</a>
0179         <span id="pcs-pdl-species-opts"></span>
0180       </div>
0181       <div>
0182         <span class="pcs-pdl-section-label">Energy:</span>
0183         <a href="#" data-filter="energy" data-value="" class="filter-link filter-active">All</a>
0184         <span id="pcs-pdl-energy-opts"></span>
0185       </div>
0186     </div>
0187
0188     <div class="pcs-pdl-summary">
0189       <strong>{% if active_release == 'all' %}All past releases{% elif active_release|slice:":4" == "all_" %}All {{ active_release|slice:"4:" }}{% else %}{{ active_release }}{% endif %}{% if active_stage %} &mdash; {% if active_stage == 'FULL' %}Simu{% else %}Reco{% endif %} only{% endif %}</strong>
0190       &middot; {{ selected_campaign_count }} campaign{{ selected_campaign_count|pluralize }}
0191       &middot; {{ tasks|length }} dataset{{ tasks|length|pluralize }}
0192       {% if aggregate_file_count or aggregate_data_size %}
0193         &middot; {{ aggregate_file_count }} files
0194         &middot; {{ aggregate_data_size|filesizeformat }}
0195       {% endif %}
0196     </div>
0197
0198     <table class="table table-sm pcs-pdl-list mb-1">
0199       <thead>
0200         <tr>
0201           <th style="width:1.5em"><input type="checkbox" id="pcs-pdl-selectall" aria-label="Select all"></th>
0202           <th style="width:3em">#</th>
0203           <th style="width:4em">Stage</th>
0204           <th style="width:5em">Release</th>
0205           <th>Dataset</th>
0206         </tr>
0207       </thead>
0208       <tbody>
0209       {% for t in tasks %}
0210         {% with output=t.overrides.past_output ds_did=t.dataset.metadata.source.location %}
0211         <tr class="pcs-pdl-row"
0212             data-detector="{{ output.filters.detector|default:'' }}"
0213             data-beam="{{ output.filters.beam|default:'' }}"
0214             data-physics="{{ output.filters.physics|default:'' }}"
0215             data-q2="{{ output.filters.q2|default:'' }}"
0216             data-species="{{ output.filters.species|default:'' }}"
0217             data-energy="{{ output.filters.energy|default:'' }}"
0218             data-incomplete="{% if not output.complete %}1{% else %}0{% endif %}"
0219             data-search="{{ ds_did|lower }}">
0220           <td><input type="checkbox" class="pcs-pdl-cb" name="task" value="{{ t.pk }}"></td>
0221           <td class="small text-muted">{{ forloop.counter }}</td>
0222           <td class="small">{% if output.stage == 'FULL' %}SIMU{% else %}{{ output.stage }}{% endif %}</td>
0223           <td class="small">{{ output.version }}</td>
0224           <td class="small">
0225             <div><strong class="text-muted">Output:</strong> {{ ds_did }}</div>
0226             <div class="text-muted">
0227               {{ t.dataset.file_count }} files
0228               &middot; {{ t.dataset.data_size|filesizeformat }}
0229               {% if output.rses %}&middot;
0230                 {% for r in output.rses %}{% if not forloop.first %}, {% endif %}{{ r.name }}{% if r.status != 'complete' %} <span class="pcs-pdl-rse-bad">({{ r.files }}/{{ r.total }})</span>{% endif %}{% endfor %}
0231               {% endif %}
0232             </div>
0233           </td>
0234         </tr>
0235         {% endwith %}
0236       {% empty %}
0237         <tr><td colspan="5" class="text-center text-muted py-3">No datasets recorded for this campaign yet &mdash; click <em>Update from epic-prod</em>.</td></tr>
0238       {% endfor %}
0239       </tbody>
0240     </table>
0241   {% elif active_lifecycle == 'last' %}
0242     {# Last not set yet — Make-last selector above is the call to action. #}
0243   {% else %}
0244     <div class="alert alert-info">
0245       No past campaigns ingested yet. Click <strong>Update from epic-prod</strong> above to populate from
0246       the cloned <code>epic-prod</code> tree.
0247     </div>
0248   {% endif %}
0249 </div>
0250
0251 <script>
0252 (function() {
0253   const all = document.getElementById('pcs-pdl-selectall');
0254   const cbs = document.querySelectorAll('.pcs-pdl-cb');
0255   if (all) all.addEventListener('change', () => cbs.forEach(c => c.checked = all.checked));
0256
0257   /* Faceted filter — Detector / Beam / Physics / Q2. Same panda-jobs
0258      convention as the Current tab's filter bar: per-filter horizontal
0259      value (count) links, counts faceted on the other active filters. */
0260   const FILTERS = ['detector','beam','physics','q2','species','energy'];
0261   const DS_KEY  = { detector:'detector', beam:'beam', physics:'physics',
0262                     q2:'q2', species:'species', energy:'energy' };
0263
0264   // Energy filter values like '500MeV' / '5GeV' need unit-aware
0265   // sort — plain numeric sort would put 5GeV (=5) before 500MeV (=500).
0266   function energyMeV(s) {
0267     const m = String(s).match(/^([\d.]+)\s*(eV|keV|MeV|GeV|TeV)$/i);
0268     if (!m) return null;
0269     const mult = {ev:1e-6, kev:1e-3, mev:1, gev:1e3, tev:1e6}[m[2].toLowerCase()];
0270     return parseFloat(m[1]) * mult;
0271   }
0272   const rows    = Array.from(document.querySelectorAll('tr.pcs-pdl-row'));
0273   if (!rows.length) return;
0274
0275   const initial = new URL(window.location.href).searchParams;
0276   const state = Object.fromEntries(FILTERS.map(k => [k, initial.get(k) || '']));
0277   state.q = (initial.get('q') || '').trim();
0278   state.incomplete = initial.get('incomplete') === '1';
0279   const qInput = document.getElementById('pcs-pdl-q');
0280   if (qInput) qInput.value = state.q;
0281
0282   function rowMatches(tr, skip) {
0283     if (skip !== 'q' && state.q) {
0284       if (!(tr.dataset.search || '').includes(state.q.toLowerCase())) return false;
0285     }
0286     if (skip !== 'incomplete' && state.incomplete) {
0287       if (tr.dataset.incomplete !== '1') return false;
0288     }
0289     for (const k of FILTERS) {
0290       if (k === skip) continue;
0291       if (state[k] && tr.dataset[DS_KEY[k]] !== state[k]) return false;
0292     }
0293     return true;
0294   }
0295
0296   function escape(s) {
0297     return String(s).replace(/[&<>"']/g, c => (
0298       {'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
0299   }
0300
0301   function renderBar() {
0302     const counts = Object.fromEntries(FILTERS.map(k => [k, {}]));
0303     let incompleteCount = 0;
0304     for (const tr of rows) {
0305       for (const k of FILTERS) {
0306         if (rowMatches(tr, k)) {
0307           const v = tr.dataset[DS_KEY[k]];
0308           if (v) counts[k][v] = (counts[k][v] || 0) + 1;
0309         }
0310       }
0311       if (rowMatches(tr, 'incomplete') && tr.dataset.incomplete === '1') {
0312         incompleteCount++;
0313       }
0314     }
0315     const incCountEl = document.querySelector('[data-count="incomplete"]');
0316     if (incCountEl) incCountEl.textContent = incompleteCount;
0317     const incLink = document.querySelector('a.filter-link[data-toggle="incomplete"]');
0318     if (incLink) incLink.classList.toggle('filter-active', !!state.incomplete);
0319     for (const k of FILTERS) {
0320       const span = document.getElementById('pcs-pdl-' + k + '-opts');
0321       if (!span) continue;
0322       const entries = Object.entries(counts[k]).sort((a, b) => {
0323         if (k === 'energy') {
0324           const ea = energyMeV(a[0]), eb = energyMeV(b[0]);
0325           if (ea !== null && eb !== null) return ea - eb;
0326         }
0327         const na = parseFloat(a[0]), nb = parseFloat(b[0]);
0328         if (!isNaN(na) && !isNaN(nb)) return na - nb;
0329         return a[0].localeCompare(b[0]);
0330       });
0331       span.innerHTML = entries.map(([v, n]) => {
0332         const active = state[k] === v ? ' filter-active' : '';
0333         return `<a href="#" data-filter="${k}" data-value="${escape(v)}"`
0334              + ` class="filter-link${active}">${escape(v)} (${n})</a>`;
0335       }).join('');
0336       const allLink = document.querySelector(
0337         `#pcs-pdl-filterbar .filter-link[data-filter="${k}"][data-value=""]`);
0338       if (allLink) allLink.classList.toggle('filter-active', !state[k]);
0339     }
0340   }
0341
0342   function syncUrl() {
0343     const u = new URL(window.location.href);
0344     const p = u.searchParams;
0345     FILTERS.forEach(k => p.delete(k));
0346     p.delete('q');
0347     p.delete('incomplete');
0348     for (const k of FILTERS) if (state[k]) p.set(k, state[k]);
0349     if (state.q) p.set('q', state.q);
0350     if (state.incomplete) p.set('incomplete', '1');
0351     history.replaceState(null, '', u.toString());
0352   }
0353
0354   function applyFilters() {
0355     for (const tr of rows) {
0356       tr.style.display = rowMatches(tr, null) ? '' : 'none';
0357     }
0358     renderBar();
0359     syncUrl();
0360   }
0361
0362   document.getElementById('pcs-pdl-filterbar')?.addEventListener('click', e => {
0363     const a = e.target.closest('a.filter-link');
0364     if (!a) return;
0365     e.preventDefault();
0366     if (a.dataset.toggle) {
0367       state[a.dataset.toggle] = !state[a.dataset.toggle];
0368     } else {
0369       const k = a.dataset.filter;
0370       const v = a.dataset.value || '';
0371       state[k] = (state[k] === v && v !== '') ? '' : v;
0372     }
0373     applyFilters();
0374   });
0375
0376   if (qInput) {
0377     qInput.addEventListener('input', () => {
0378       state.q = qInput.value.trim();
0379       applyFilters();
0380     });
0381   }
0382   document.getElementById('pcs-pdl-clear')?.addEventListener('click', () => {
0383     state.q = '';
0384     state.incomplete = false;
0385     if (qInput) qInput.value = '';
0386     for (const k of FILTERS) state[k] = '';
0387     applyFilters();
0388   });
0389
0390   applyFilters();
0391 })();
0392 </script>
0393
0394 {% if active_lifecycle == 'last' and rucio_timeline_json != 'null' %}
0395 <script src="https://cdn.plot.ly/plotly-2.35.2.min.js"></script>
0396 <script>
0397 (function() {
0398   const data = {{ rucio_timeline_json|safe }};
0399   const el = document.getElementById('pcs-rucio-timeline');
0400   if (!el) return;
0401   if (!data || !data.dates || !data.dates.length) {
0402     el.innerHTML = '<div class="text-muted small">No Rucio arrivals timeline available — '
0403       + 'click <em>Update from Rucio</em> to populate the snapshot.</div>';
0404     return;
0405   }
0406   const TB = 1e12;
0407   const SIMU_COLOR = '#1f77b4';
0408   const RECO_COLOR = '#ff7f0e';
0409   const tb = bytes => bytes.map(b => b / TB);
0410   const traces = [
0411     {x: data.dates, y: data.simu.cum_datasets, name: 'Simu',
0412      legendgroup: 'simu', mode: 'lines', line: {shape: 'hv', color: SIMU_COLOR},
0413      xaxis: 'x', yaxis: 'y'},
0414     {x: data.dates, y: data.reco.cum_datasets, name: 'Reco',
0415      legendgroup: 'reco', mode: 'lines', line: {shape: 'hv', color: RECO_COLOR},
0416      xaxis: 'x', yaxis: 'y'},
0417     {x: data.dates, y: data.simu.cum_files, legendgroup: 'simu',
0418      mode: 'lines', line: {shape: 'hv', color: SIMU_COLOR},
0419      xaxis: 'x', yaxis: 'y2', showlegend: false},
0420     {x: data.dates, y: data.reco.cum_files, legendgroup: 'reco',
0421      mode: 'lines', line: {shape: 'hv', color: RECO_COLOR},
0422      xaxis: 'x', yaxis: 'y2', showlegend: false},
0423     {x: data.dates, y: tb(data.simu.cum_bytes), legendgroup: 'simu',
0424      mode: 'lines', line: {shape: 'hv', color: SIMU_COLOR},
0425      xaxis: 'x', yaxis: 'y3', showlegend: false},
0426     {x: data.dates, y: tb(data.reco.cum_bytes), legendgroup: 'reco',
0427      mode: 'lines', line: {shape: 'hv', color: RECO_COLOR},
0428      xaxis: 'x', yaxis: 'y3', showlegend: false},
0429   ];
0430   const layout = {
0431     title: {text: 'Rucio arrivals — ' + (data.campaign_name || ''),
0432             font: {size: 14}},
0433     margin: {l: 70, r: 30, t: 40, b: 40},
0434     height: 540,
0435     xaxis:  {anchor: 'y3', type: 'date'},
0436     yaxis:  {domain: [0.70, 1.00], title: 'datasets', rangemode: 'tozero'},
0437     yaxis2: {domain: [0.36, 0.66], title: 'files (registered)',
0438              rangemode: 'tozero'},
0439     yaxis3: {domain: [0.00, 0.32], title: 'output (TB)', rangemode: 'tozero'},
0440     legend: {orientation: 'h', y: -0.12},
0441     showlegend: true,
0442   };
0443   Plotly.newPlot(el, traces, layout, {displayModeBar: false, responsive: true});
0444 })();
0445 </script>
0446 {% endif %}
0447 {% endblock %}