monitor_app/viewdir/pandamon.py

0001 """
0002 PanDA Production Monitor views.
0003
0004 Web views for ePIC PanDA production monitoring — jobs, tasks, errors,
0005 activity overview, and detail pages with rich cross-linking.
0006 """
0007
0008 from django.shortcuts import render
0009 from django.contrib.auth.decorators import login_required
0010 from django.http import JsonResponse, HttpResponse
0011 from django.urls import reverse
0012
0013 from datetime import datetime
0014 from zoneinfo import ZoneInfo
0015
0016 from ..utils import DataTablesProcessor
0017 from ..panda import (
0018     get_activity, study_job, list_jobs,
0019     list_jobs_dt, list_tasks_dt,
0020     job_filter_counts, task_filter_counts,
0021     get_task, error_summary, diagnose_jobs,
0022     list_queues, get_queue,
0023 )
0024 from ..panda.constants import LIST_FIELDS, TASK_LIST_FIELDS
0025
0026
0027 # ── Column definitions ───────────────────────────────────────────────────────
0028
0029 JOB_COLUMNS = [
0030     {'name': 'pandaid', 'title': 'PanDA ID', 'orderable': True},
0031     {'name': 'jeditaskid', 'title': 'Task ID', 'orderable': True},
0032     {'name': 'produsername', 'title': 'User', 'orderable': True},
0033     {'name': 'jobstatus', 'title': 'Status', 'orderable': True},
0034     {'name': 'computingsite', 'title': 'Site', 'orderable': True},
0035     {'name': 'transformation', 'title': 'Transformation', 'orderable': True},
0036     {'name': 'creationtime', 'title': 'Created', 'orderable': True},
0037     {'name': 'endtime', 'title': 'Ended', 'orderable': True},
0038     {'name': 'corecount', 'title': 'Cores', 'orderable': True},
0039 ]
0040
0041 JOB_FIELD_NAMES = [c['name'] for c in JOB_COLUMNS]
0042
0043 # Map DataTables column index to SQL ORDER BY expression
0044 JOB_ORDER_MAP = {
0045     0: '"pandaid"', 1: '"jeditaskid"', 2: '"produsername"',
0046     3: '"jobstatus"', 4: '"computingsite"', 5: '"transformation"',
0047     6: '"creationtime"', 7: '"endtime"', 8: '"corecount"',
0048 }
0049
0050 TASK_COLUMNS = [
0051     {'name': 'jeditaskid', 'title': 'Task ID', 'orderable': True},
0052     {'name': 'taskname', 'title': 'Task Name', 'orderable': True},
0053     {'name': 'status', 'title': 'Status', 'orderable': True},
0054     {'name': 'username', 'title': 'User', 'orderable': True},
0055     {'name': 'workinggroup', 'title': 'Working Group', 'orderable': True},
0056     {'name': 'creationdate', 'title': 'Created', 'orderable': True},
0057     {'name': 'modificationtime', 'title': 'Modified', 'orderable': True},
0058     {'name': 'progress', 'title': 'Progress', 'orderable': True},
0059     {'name': 'failurerate', 'title': 'Failure Rate', 'orderable': True},
0060 ]
0061
0062 TASK_FIELD_NAMES = [c['name'] for c in TASK_COLUMNS]
0063
0064 TASK_ORDER_MAP = {
0065     0: '"jeditaskid"', 1: '"taskname"', 2: '"status"',
0066     3: '"username"', 4: '"workinggroup"', 5: '"creationdate"',
0067     6: '"modificationtime"', 7: '"progress"', 8: '"failurerate"',
0068 }
0069
0070 ERROR_COLUMNS = [
0071     {'name': 'error_source', 'title': 'Component', 'orderable': False},
0072     {'name': 'error_code', 'title': 'Code', 'orderable': False},
0073     {'name': 'error_diag', 'title': 'Diagnostic', 'orderable': False},
0074     {'name': 'count', 'title': 'Count', 'orderable': False},
0075     {'name': 'task_count', 'title': 'Tasks', 'orderable': False},
0076     {'name': 'users', 'title': 'Users', 'orderable': False},
0077     {'name': 'sites', 'title': 'Sites', 'orderable': False},
0078 ]
0079
0080 DIAG_COLUMNS = [
0081     {'name': 'pandaid', 'title': 'PanDA ID', 'orderable': False},
0082     {'name': 'jeditaskid', 'title': 'Task ID', 'orderable': False},
0083     {'name': 'produsername', 'title': 'User', 'orderable': False},
0084     {'name': 'jobstatus', 'title': 'Status', 'orderable': False},
0085     {'name': 'computingsite', 'title': 'Site', 'orderable': False},
0086     {'name': 'errors', 'title': 'Errors', 'orderable': False},
0087     {'name': 'endtime', 'title': 'Ended', 'orderable': False},
0088 ]
0089
0090
0091 # ── Helpers ──────────────────────────────────────────────────────────────────
0092
0093 _EASTERN = ZoneInfo('America/New_York')
0094
0095
0096 def _linkify(text):
0097     """Wrap text in an <a> tag if it looks like a URL.
0098
0099     TRF links (pandaserver-doma.cern.ch/trf/) are routed through our
0100     view-text endpoint which extracts readable content from self-extracting zips.
0101     """
0102     if text and text.startswith(('http://', 'https://')):
0103         href = text
0104         if 'pandaserver-doma.cern.ch/trf/' in text:
0105             href = reverse('monitor_app:panda_view_text') + '?url=' + text
0106         return f'<a href="{href}" target="_blank" rel="noopener">{text}</a>'
0107     return text
0108
0109
0110 def _fmt_dt(val):
0111     """Format an ISO datetime string or datetime object for display."""
0112     if not val:
0113         return ''
0114     if isinstance(val, str):
0115         try:
0116             val = datetime.fromisoformat(val)
0117         except (ValueError, TypeError):
0118             return val
0119     return val.astimezone(_EASTERN).strftime('%Y%m%d %H:%M:%S')
0120
0121
0122 STATUS_COLORS = {
0123     'finished': '#28a745', 'done': '#28a745',
0124     'failed': '#dc3545', 'broken': '#dc3545',
0125     'running': '#007bff',
0126     'activated': '#17a2b8', 'ready': '#17a2b8',
0127     'cancelled': '#6c757d', 'aborted': '#6c757d', 'closed': '#6c757d',
0128     'exhausted': '#fd7e14',
0129 }
0130
0131
0132 def _status_badge(status, url=None):
0133     """Render a status as a colored badge, optionally linked."""
0134     color = STATUS_COLORS.get(status, '#6c757d')
0135     badge = (f'<span style="background:{color};color:#fff;padding:2px 8px;'
0136              f'border-radius:3px;font-size:0.85em;">{status}</span>')
0137     if url:
0138         return f'<a href="{url}" style="text-decoration:none;">{badge}</a>'
0139     return badge
0140
0141
0142 DAYS_OPTIONS = [1, 3, 7, 14, 30]
0143
0144
0145 def _get_days(request):
0146     """Extract days parameter from request, default 7."""
0147     try:
0148         return int(request.GET.get('days', 7))
0149     except (ValueError, TypeError):
0150         return 7
0151
0152
0153 def _days_context(days):
0154     """Build days selector context for templates."""
0155     return {
0156         'days': days,
0157         'days_options': [{'value': d, 'active': d == days} for d in DAYS_OPTIONS],
0158     }
0159
0160
0161 # ── Activity overview ────────────────────────────────────────────────────────
0162
0163 @login_required
0164 def panda_activity(request):
0165     days = _get_days(request)
0166     data = get_activity(days=days)
0167     if 'error' in data:
0168         ctx = {'error': data['error']}
0169         ctx.update(_days_context(days))
0170         return render(request, 'monitor_app/panda_activity.html', ctx)
0171     data.update(_days_context(days))
0172     return render(request, 'monitor_app/panda_activity.html', data)
0173
0174
0175 # ── Job list ─────────────────────────────────────────────────────────────────
0176
0177 @login_required
0178 def panda_jobs_list(request):
0179     days = _get_days(request)
0180     selected_site = request.GET.get('site', '')
0181     description = f'Production jobs from the last {days} days.'
0182     if selected_site:
0183         site_url = reverse('monitor_app:epic_queue_detail', args=[selected_site])
0184         description += f'<br><a href="{site_url}">Site info for <strong>{selected_site}</strong></a>'
0185
0186     context = {
0187         'table_title': 'PanDA Jobs',
0188         'table_description': description,
0189         'ajax_url': reverse('monitor_app:panda_jobs_datatable_ajax'),
0190         'filter_counts_url': reverse('monitor_app:panda_jobs_filter_counts'),
0191         'columns': JOB_COLUMNS,
0192         'filter_fields': [
0193             {'name': 'status', 'label': 'Status', 'type': 'select'},
0194             {'name': 'username', 'label': 'User', 'type': 'select'},
0195             {'name': 'site', 'label': 'Site', 'type': 'select'},
0196         ],
0197         'selected_status': request.GET.get('status', ''),
0198         'selected_username': request.GET.get('username', ''),
0199         'selected_site': request.GET.get('site', ''),
0200         'selected_taskid': request.GET.get('taskid', ''),
0201     }
0202     context.update(_days_context(days))
0203     return render(request, 'monitor_app/panda_jobs_list.html', context)
0204
0205
0206 @login_required
0207 def panda_jobs_datatable_ajax(request):
0208     dt = DataTablesProcessor(request, JOB_FIELD_NAMES,
0209                              default_order_column=0, default_order_direction='desc')
0210     days = _get_days(request)
0211     status = request.GET.get('status', '') or None
0212     username = request.GET.get('username', '') or None
0213     site = request.GET.get('site', '') or None
0214     taskid = request.GET.get('taskid', '') or None
0215     reqid = request.GET.get('reqid', '') or None
0216
0217     order_col = JOB_ORDER_MAP.get(dt.order_column_idx, '"pandaid"')
0218     order_dir = 'ASC' if dt.order_direction == 'asc' else 'DESC'
0219     order_by = f'{order_col} {order_dir}'
0220
0221     rows, total, filtered = list_jobs_dt(
0222         days=days, status=status, username=username, site=site,
0223         taskid=taskid, reqid=reqid,
0224         order_by=order_by, limit=dt.length, offset=dt.start,
0225         search=dt.search_value or None,
0226     )
0227
0228     data = []
0229     for job in rows:
0230         job_url = reverse('monitor_app:panda_job_detail', args=[job['pandaid']])
0231         task_url = reverse('monitor_app:panda_task_detail', args=[job['jeditaskid']]) if job.get('jeditaskid') else None
0232         jobs_by_user_url = reverse('monitor_app:panda_jobs_list') + f'?days={days}&username={job["produsername"]}' if job.get('produsername') else None
0233         jobs_by_site_url = reverse('monitor_app:panda_jobs_list') + f'?days={days}&site={job["computingsite"]}' if job.get('computingsite') else None
0234         jobs_by_status_url = reverse('monitor_app:panda_jobs_list') + f'?days={days}&status={job["jobstatus"]}' if job.get('jobstatus') else None
0235
0236         data.append([
0237             f'<a href="{job_url}">{job["pandaid"]}</a>',
0238             f'<a href="{task_url}">{job["jeditaskid"]}</a>' if task_url else str(job.get('jeditaskid', '')),
0239             f'<a href="{jobs_by_user_url}">{job["produsername"]}</a>' if jobs_by_user_url else '',
0240             _status_badge(job['jobstatus'], jobs_by_status_url) if job.get('jobstatus') else '',
0241             f'<a href="{jobs_by_site_url}">{job["computingsite"]}</a>' if jobs_by_site_url else '',
0242             _linkify(job.get('transformation', '') or ''),
0243             _fmt_dt(job.get('creationtime')),
0244             _fmt_dt(job.get('endtime')),
0245             str(job.get('corecount', '') or ''),
0246         ])
0247
0248     return dt.create_response(data, total, filtered)
0249
0250
0251 @login_required
0252 def panda_jobs_filter_counts(request):
0253     days = _get_days(request)
0254     status = request.GET.get('status', '') or None
0255     username = request.GET.get('username', '') or None
0256     site = request.GET.get('site', '') or None
0257     taskid = request.GET.get('taskid', '') or None
0258     reqid = request.GET.get('reqid', '') or None
0259
0260     counts = job_filter_counts(days=days, status=status, username=username,
0261                                site=site, taskid=taskid, reqid=reqid)
0262     return JsonResponse({'filter_counts': counts})
0263
0264
0265 # ── Task list ────────────────────────────────────────────────────────────────
0266
0267 @login_required
0268 def panda_tasks_list(request):
0269     days = _get_days(request)
0270     context = {
0271         'table_title': 'PanDA Tasks',
0272         'table_description': f'JEDI tasks from the last {days} days.',
0273         'ajax_url': reverse('monitor_app:panda_tasks_datatable_ajax'),
0274         'filter_counts_url': reverse('monitor_app:panda_tasks_filter_counts'),
0275         'columns': TASK_COLUMNS,
0276         'filter_fields': [
0277             {'name': 'status', 'label': 'Status', 'type': 'select'},
0278             {'name': 'username', 'label': 'User', 'type': 'select'},
0279             {'name': 'workinggroup', 'label': 'Working Group', 'type': 'select'},
0280         ],
0281         'selected_status': request.GET.get('status', ''),
0282         'selected_username': request.GET.get('username', ''),
0283         'selected_workinggroup': request.GET.get('workinggroup', ''),
0284     }
0285     context.update(_days_context(days))
0286     return render(request, 'monitor_app/panda_tasks_list.html', context)
0287
0288
0289 @login_required
0290 def panda_tasks_datatable_ajax(request):
0291     dt = DataTablesProcessor(request, TASK_FIELD_NAMES,
0292                              default_order_column=0, default_order_direction='desc')
0293     days = _get_days(request)
0294     status = request.GET.get('status', '') or None
0295     username = request.GET.get('username', '') or None
0296     taskname = request.GET.get('taskname', '') or None
0297     workinggroup = request.GET.get('workinggroup', '') or None
0298
0299     order_col = TASK_ORDER_MAP.get(dt.order_column_idx, '"jeditaskid"')
0300     order_dir = 'ASC' if dt.order_direction == 'asc' else 'DESC'
0301     order_by = f'{order_col} {order_dir}'
0302
0303     rows, total, filtered = list_tasks_dt(
0304         days=days, status=status, username=username, taskname=taskname,
0305         workinggroup=workinggroup,
0306         order_by=order_by, limit=dt.length, offset=dt.start,
0307         search=dt.search_value or None,
0308     )
0309
0310     data = []
0311     for task in rows:
0312         task_url = reverse('monitor_app:panda_task_detail', args=[task['jeditaskid']])
0313         tasks_by_user_url = reverse('monitor_app:panda_tasks_list') + f'?days={days}&username={task["username"]}' if task.get('username') else None
0314         tasks_by_status_url = reverse('monitor_app:panda_tasks_list') + f'?days={days}&status={task["status"]}' if task.get('status') else None
0315         tasks_by_wg_url = reverse('monitor_app:panda_tasks_list') + f'?days={days}&workinggroup={task["workinggroup"]}' if task.get('workinggroup') else None
0316
0317         # Truncate taskname for display
0318         taskname_display = task.get('taskname', '') or ''
0319         if len(taskname_display) > 80:
0320             taskname_display = taskname_display[:77] + '...'
0321
0322         progress = task.get('progress')
0323         progress_str = f'{progress}%' if progress is not None else ''
0324
0325         failurerate = task.get('failurerate')
0326         failurerate_str = f'{failurerate}%' if failurerate is not None else ''
0327
0328         data.append([
0329             f'<a href="{task_url}">{task["jeditaskid"]}</a>',
0330             f'<a href="{task_url}" title="{task.get("taskname", "")}">{taskname_display}</a>',
0331             _status_badge(task['status'], tasks_by_status_url) if task.get('status') else '',
0332             f'<a href="{tasks_by_user_url}">{task["username"]}</a>' if tasks_by_user_url else '',
0333             f'<a href="{tasks_by_wg_url}">{task["workinggroup"]}</a>' if tasks_by_wg_url else str(task.get('workinggroup', '') or ''),
0334             _fmt_dt(task.get('creationdate')),
0335             _fmt_dt(task.get('modificationtime')),
0336             progress_str,
0337             failurerate_str,
0338         ])
0339
0340     return dt.create_response(data, total, filtered)
0341
0342
0343 @login_required
0344 def panda_tasks_filter_counts(request):
0345     days = _get_days(request)
0346     status = request.GET.get('status', '') or None
0347     username = request.GET.get('username', '') or None
0348     workinggroup = request.GET.get('workinggroup', '') or None
0349
0350     counts = task_filter_counts(days=days, status=status,
0351                                 username=username, workinggroup=workinggroup)
0352     return JsonResponse({'filter_counts': counts})
0353
0354
0355 # ── Job detail ───────────────────────────────────────────────────────────────
0356
0357 @login_required
0358 def panda_job_detail(request, pandaid):
0359     data = study_job(int(pandaid))
0360     if 'error' in data:
0361         return render(request, 'monitor_app/panda_job_detail.html',
0362                       {'error': data['error'], 'pandaid': pandaid})
0363     data['pandaid'] = pandaid
0364     job = data.get('job') or {}
0365     job['transformation_is_url'] = (job.get('transformation') or '').startswith(('http://', 'https://'))
0366     trf = job.get('transformation') or ''
0367     if 'pandaserver-doma.cern.ch/trf/' in trf:
0368         job['transformation_view_url'] = reverse('monitor_app:panda_view_text') + '?url=' + trf
0369     return render(request, 'monitor_app/panda_job_detail.html', data)
0370
0371
0372 # ── View text (transformation script viewer) ────────────────────────────────
0373
0374 def panda_view_text(request):
0375     """Fetch a PanDA transformation URL — self-extracting zip with embedded scripts.
0376
0377     Extracts the bash header and all text files from the zip, presents them
0378     as readable plain text.
0379     """
0380     import httpx
0381     import io
0382     import zipfile
0383     url = request.GET.get('url', '')
0384     if not url or not url.startswith('https://'):
0385         return HttpResponse('Missing or invalid url parameter', status=400,
0386                             content_type='text/plain')
0387     try:
0388         resp = httpx.get(url, timeout=15, follow_redirects=True)
0389     except Exception as e:
0390         return HttpResponse(f'Failed to fetch: {e}', status=502,
0391                             content_type='text/plain')
0392     data = resp.content
0393     parts = []
0394     # Extract the bash header (text before binary zip data)
0395     try:
0396         lines = []
0397         for line in data.split(b'\n'):
0398             try:
0399                 lines.append(line.decode('utf-8'))
0400             except UnicodeDecodeError:
0401                 break
0402         if lines:
0403             parts.append(f'=== Shell header ({len(lines)} lines) ===\n')
0404             parts.append('\n'.join(lines))
0405     except Exception:
0406         pass
0407     # Extract text files from the zip
0408     try:
0409         buf = io.BytesIO(data)
0410         with zipfile.ZipFile(buf) as zf:
0411             for name in zf.namelist():
0412                 try:
0413                     content = zf.read(name).decode('utf-8')
0414                     parts.append(f'\n\n=== {name} ===\n')
0415                     parts.append(content)
0416                 except (UnicodeDecodeError, KeyError):
0417                     parts.append(f'\n\n=== {name} (binary, skipped) ===\n')
0418     except zipfile.BadZipFile:
0419         if not parts:
0420             # Not a zip, just serve as text
0421             parts.append(data.decode('utf-8', errors='replace'))
0422     return HttpResponse(''.join(parts), content_type='text/plain; charset=utf-8')
0423
0424
0425 # ── Task detail ──────────────────────────────────────────────────────────────
0426
0427 @login_required
0428 def panda_task_detail(request, jeditaskid):
0429     task = get_task(int(jeditaskid))
0430     if isinstance(task, dict) and 'error' in task:
0431         return render(request, 'monitor_app/panda_task_detail.html',
0432                       {'error': task['error'], 'jeditaskid': jeditaskid})
0433
0434     # Get jobs for this task
0435     jobs_data = list_jobs(taskid=int(jeditaskid), days=90, limit=200)
0436     jobs = jobs_data.get('jobs', []) if not jobs_data.get('error') else []
0437     summary = jobs_data.get('summary', {}) if not jobs_data.get('error') else {}
0438
0439     return render(request, 'monitor_app/panda_task_detail.html', {
0440         'task': task,
0441         'jeditaskid': jeditaskid,
0442         'jobs': jobs,
0443         'job_summary': summary,
0444         'job_count': len(jobs),
0445     })
0446
0447
0448 # ── Error summary ────────────────────────────────────────────────────────────
0449
0450 @login_required
0451 def panda_errors_list(request):
0452     days = _get_days(request)
0453     context = {
0454         'table_title': 'PanDA Error Summary',
0455         'table_description': f'Top error patterns across failed jobs in the last {days} days.',
0456         'ajax_url': reverse('monitor_app:panda_errors_datatable_ajax'),
0457         'columns': ERROR_COLUMNS,
0458     }
0459     context.update(_days_context(days))
0460     return render(request, 'monitor_app/panda_errors.html', context)
0461
0462
0463 @login_required
0464 def panda_errors_datatable_ajax(request):
0465     dt = DataTablesProcessor(request, [c['name'] for c in ERROR_COLUMNS],
0466                              default_order_column=3, default_order_direction='desc')
0467     days = _get_days(request)
0468     username = request.GET.get('username', '') or None
0469     site = request.GET.get('site', '') or None
0470     error_source = request.GET.get('error_source', '') or None
0471
0472     result = error_summary(days=days, username=username, site=site,
0473                            error_source=error_source, limit=200)
0474
0475     if 'error' in result:
0476         return dt.create_response([], 0, 0)
0477
0478     errors = result.get('errors', [])
0479     total = len(errors)
0480
0481     data = []
0482     for err in errors:
0483         diag_url = reverse('monitor_app:panda_diagnostics_list') + f'?days={days}&error_source={err["error_source"]}'
0484         users_str = ', '.join(err.get('users', [])[:5])
0485         if len(err.get('users', [])) > 5:
0486             users_str += f' (+{len(err["users"]) - 5})'
0487         sites_str = ', '.join(err.get('sites', [])[:3])
0488         if len(err.get('sites', [])) > 3:
0489             sites_str += f' (+{len(err["sites"]) - 3})'
0490
0491         diag_text = err.get('error_diag', '') or ''
0492         if len(diag_text) > 120:
0493             diag_text = diag_text[:117] + '...'
0494
0495         data.append([
0496             f'<a href="{diag_url}">{err["error_source"]}</a>',
0497             str(err.get('error_code', '')),
0498             f'<span title="{err.get("error_diag", "")}">{diag_text}</span>',
0499             str(err.get('count', 0)),
0500             str(err.get('task_count', 0)),
0501             users_str,
0502             sites_str,
0503         ])
0504
0505     return dt.create_response(data, total, total)
0506
0507
0508 # ── Diagnostics ──────────────────────────────────────────────────────────────
0509
0510 @login_required
0511 def panda_diagnostics_list(request):
0512     days = _get_days(request)
0513     context = {
0514         'table_title': 'PanDA Job Diagnostics',
0515         'table_description': f'Failed jobs with error details from the last {days} days.',
0516         'ajax_url': reverse('monitor_app:panda_diagnostics_datatable_ajax'),
0517         'columns': DIAG_COLUMNS,
0518     }
0519     context.update(_days_context(days))
0520     return render(request, 'monitor_app/panda_diagnostics.html', context)
0521
0522
0523 @login_required
0524 def panda_diagnostics_datatable_ajax(request):
0525     dt = DataTablesProcessor(request, [c['name'] for c in DIAG_COLUMNS],
0526                              default_order_column=0, default_order_direction='desc')
0527     days = _get_days(request)
0528     username = request.GET.get('username', '') or None
0529     site = request.GET.get('site', '') or None
0530     taskid = request.GET.get('taskid', '') or None
0531     error_source = request.GET.get('error_source', '') or None
0532
0533     result = diagnose_jobs(days=days, username=username, site=site,
0534                            taskid=taskid, error_component=error_source,
0535                            limit=500)
0536
0537     if 'error' in result:
0538         return dt.create_response([], 0, 0)
0539
0540     jobs = result.get('jobs', [])
0541     total = len(jobs)
0542
0543     # Apply pagination
0544     page_jobs = jobs[dt.start:dt.start + dt.length]
0545
0546     data = []
0547     for job in page_jobs:
0548         job_url = reverse('monitor_app:panda_job_detail', args=[job['pandaid']])
0549         task_url = reverse('monitor_app:panda_task_detail', args=[job['jeditaskid']]) if job.get('jeditaskid') else None
0550
0551         errors_html = []
0552         for err in job.get('errors', []):
0553             diag = err.get('diag', '')
0554             if len(diag) > 80:
0555                 diag = diag[:77] + '...'
0556             errors_html.append(f'<strong>{err["component"]}</strong>:{err["code"]} {diag}')
0557
0558         data.append([
0559             f'<a href="{job_url}">{job["pandaid"]}</a>',
0560             f'<a href="{task_url}">{job["jeditaskid"]}</a>' if task_url else str(job.get('jeditaskid', '')),
0561             job.get('produsername', ''),
0562             _status_badge(job['jobstatus']) if job.get('jobstatus') else '',
0563             job.get('computingsite', ''),
0564             '<br>'.join(errors_html) if errors_html else '',
0565             _fmt_dt(job.get('endtime')),
0566         ])
0567
0568     return dt.create_response(data, total, total)
0569
0570
0571 # ── ePIC Queue views ────────────────────────────────────────────────────────
0572
0573 @login_required
0574 def epic_queues_list(request):
0575     """ePIC compute queues from live PanDA schedconfig."""
0576     result = list_queues(vo='eic')
0577     queues = result.get('queues', [])
0578     return render(request, 'monitor_app/epic_queues_list.html', {
0579         'queues': queues,
0580     })
0581
0582
0583 @login_required
0584 def epic_queue_detail(request, queue_name):
0585     """Full schedconfig for a single ePIC queue."""
0586     import json as json_mod
0587     result = get_queue(queue_name)
0588     if 'error' in result:
0589         return render(request, 'monitor_app/epic_queue_detail.html', {
0590             'error': result['error'],
0591             'queue_name': queue_name,
0592         })
0593     config = result['queue']
0594
0595     # Separate into sections for readability
0596     identity_keys = [
0597         'panda_queue', 'name', 'nickname', 'siteid', 'site', 'panda_site',
0598         'atlas_site', 'gocname', 'id',
0599     ]
0600     status_keys = [
0601         'status', 'state', 'rc_site_state', 'state_comment', 'state_update',
0602         'last_modified', 'last_update',
0603     ]
0604     resource_keys = [
0605         'resource_type', 'type', 'capability', 'corecount', 'corepower',
0606         'maxrss', 'meanrss', 'minrss', 'maxtime', 'mintime', 'maxwdir',
0607         'maxinputsize', 'timefloor', 'vo_name',
0608     ]
0609     location_keys = [
0610         'region', 'country', 'cloud', 'tier', 'tier_level', 'rc', 'rc_site',
0611         'rc_country',
0612     ]
0613     container_keys = [
0614         'container_type', 'container_options', 'is_cvmfs',
0615     ]
0616     pilot_keys = [
0617         'pilot_version', 'pilot_manager', 'python_version', 'jobseed',
0618     ]
0619
0620     def _section(keys):
0621         return {k: config[k] for k in keys if k in config}
0622
0623     sections = [
0624         ('Identity', _section(identity_keys)),
0625         ('Status', _section(status_keys)),
0626         ('Resources', _section(resource_keys)),
0627         ('Location', _section(location_keys)),
0628         ('Container', _section(container_keys)),
0629         ('Pilot', _section(pilot_keys)),
0630     ]
0631
0632     # Everything else goes in "Other"
0633     shown = set()
0634     for _, s in sections:
0635         shown.update(s.keys())
0636     other = {k: v for k, v in config.items() if k not in shown}
0637
0638     return render(request, 'monitor_app/epic_queue_detail.html', {
0639         'queue_name': queue_name,
0640         'sections': sections,
0641         'other': other,
0642         'config_json': json_mod.dumps(config, indent=2, default=str),
0643     })