File indexing completed on 2026-09-01 09:34:23
0001 """PanDA error-code labels by error component.
0002
0003 PILOT_LABELS is generated from the pilot error catalog by
0004 scripts/gen-pilot-error-labels.py; edit that generator, not this table.
0005 TASKBUFFER_LABELS transcribes the PanDA server's own definitions
0006 (pandaserver/taskbuffer/ErrorCode.py) — the component names the
0007 recording server module, not a cause: taskbuffer 102 means the server
0008 expired the job, most often one that waited past its limit without
0009 being dispatched.
0010 category_label() is the shared renderer for component:code categories.
0011 """
0012
0013 TASKBUFFER_LABELS = {
0014 100: "Killed",
0015 101: "Transfer timeout",
0016 102: "Expired by the server (waited past its limit)",
0017 103: "Aborted",
0018 105: "Reassigned by re-brokerage",
0019 106: "Reassigned by server-side retry",
0020 107: "Retried by pilot",
0021 111: "Retried for event service",
0022 112: "Event service merge",
0023 113: "Merge job failed",
0024 114: "Event service max attempts reached",
0025 115: "Waiting on other event consumers",
0026 116: "Killed as unused event consumer",
0027 117: "No events processed on worker",
0028 118: "No events processed, last consumer",
0029 119: "All event ranges failed",
0030 120: "Consumer generated event-service merge",
0031 121: "Associated event consumer failed",
0032 122: "Killed for preemption",
0033 123: "Retried without processing events",
0034 124: "Input files inconsistent with JEDI",
0035 125: "No event service queues available",
0036 126: "Closed in bad job status",
0037 130: "Fast re-brokerage when overloaded",
0038 200: "Job cloning semaphore lock failed",
0039 300: "Worker finished before the job",
0040 403: "Forbidden request",
0041 404: "File not found",
0042 }
0043
0044 PILOT_LABELS = {
0045 1008: "General pilot error, consult batch log",
0046 1098: "Not enough local space",
0047 1099: "Failed to stage-in file",
0048 1100: "Replica not found",
0049 1103: "No such file or directory",
0050 1104: "User work directory too large",
0051 1106: "Payload log or stdout file too big",
0052 1110: "Failed during payload setup",
0053 1115: "NFS SQLite locking problems",
0054 1116: "Pilot could not download queuedata",
0055 1117: "Pilot found non-valid queuedata",
0056 1124: "Output file too large",
0057 1133: "Fetching default storage failed: no activity related storage defined",
0058 1137: "Failed to stage-out file",
0059 1141: "md5sum mismatch on output file",
0060 1143: "Failed to chmod transform",
0061 1144: "This job was killed by panda server",
0062 1145: "md5sum mismatch on input file",
0063 1149: "Transform could not be downloaded",
0064 1150: "Looping job killed by pilot",
0065 1163: "Grid proxy not valid",
0066 1165: "Local output file is missing",
0067 1168: "Total file size too large",
0068 1171: "adler32 mismatch on input file",
0069 1172: "adler32 mismatch on output file",
0070 1177: "Voms proxy not valid",
0071 1180: "Globus system error during stage-in",
0072 1181: "Globus system error during stage-out",
0073 1186: "Software directory does not exist",
0074 1187: "Payload metadata does not exist",
0075 1190: "LFN too long (exceeding limit of 255 characters)",
0076 1191: "File size cannot be zero",
0077 1199: "Failed to create local directory",
0078 1200: "Job terminated by unknown kill signal",
0079 1201: "Job killed by signal: SIGTERM",
0080 1202: "Job killed by signal: SIGQUIT",
0081 1203: "Job killed by signal: SIGSEGV",
0082 1204: "Job killed by signal: SIGXCPU",
0083 1206: "Job killed by signal: SIGBUS",
0084 1207: "Job killed by signal: SIGUSR1",
0085 1211: "Missing installation",
0086 1212: "Payload ran out of memory",
0087 1213: "Reached batch system time limit",
0088 1220: "Job failed due to unknown reason (consult log file)",
0089 1221: "File already exists",
0090 1223: "Transform failed due to bad_alloc",
0091 1224: "Event service: recoverable error",
0092 1228: "Event service: fatal error",
0093 1234: "Clone job is already executed",
0094 1235: "Payload exceeded maximum allowed memory",
0095 1236: "Failed by server",
0096 1238: "Event service: no events",
0097 1240: "Failed to handle message from payload",
0098 1242: "Query checksum is not supported",
0099 1244: "No release candidates found",
0100 1246: "User tarball could not be downloaded from PanDA server",
0101 1247: "Badly formed XML",
0102 1300: "The class or function is not implemented",
0103 1301: "An unknown pilot exception has occurred",
0104 1302: "Failed to convert object data",
0105 1303: "Failed during file handling",
0106 1305: "Failed to execute payload",
0107 1306: "Singularity: general failure",
0108 1307: "Singularity: No more available loop devices",
0109 1308: "Singularity: Not mounting requested bind point",
0110 1309: "Singularity: Failed to mount image",
0111 1310: "Exception caught during payload execution",
0112 1311: "Not defined",
0113 1312: "Not same length",
0114 1313: "No protocol defined for storage endpoint",
0115 1314: "Unknown checksum type",
0116 1315: "Unknown transform failure",
0117 1316: "Rucio: Service unavailable",
0118 1317: "Exceeded maximum waiting time",
0119 1318: "Failed to communicate with server",
0120 1319: "An internal Pilot problem has occurred (consult Pilot log)",
0121 1320: "Failed during creation of log file",
0122 1321: "Failed to get client location for Rucio",
0123 1322: "Failed to get replicas from Rucio",
0124 1323: "Unknown copy tool",
0125 1324: "Service not available at the moment",
0126 1325: "Singularity: not installed",
0127 1326: "No matching replicas were found in list_replicas() output",
0128 1327: "Unable to stage-in file since network is unreachable",
0129 1328: "SIGSEGV: Invalid memory reference or a segmentation fault",
0130 1329: "Failed to construct SURL for non-deterministic ddm (update CRIC)",
0131 1330: "JSON retrieval timed out",
0132 1331: "Input file is missing in storage element",
0133 1332: "Black hole detected in file system (consult Pilot log)",
0134 1333: "No space left on device",
0135 1334: "Setup failed with a fatal exception (consult Payload log)",
0136 1335: "User code not available on PanDA server (resubmit task with --useNewCode)",
0137 1336: "Job is already running elsewhere",
0138 1337: "Memory monitor produced bad output",
0139 1338: "Authentication failure during stage-in",
0140 1339: "Local DBRelease handling failed (consult Pilot log)",
0141 1340: "Singularity: Failed invoking the NEWUSER namespace runtime",
0142 1341: "Bad queue configuration detected",
0143 1342: "Failed to import middleware (consult Pilot log)",
0144 1343: "Found no output in job report",
0145 1344: "Resource temporarily unavailable",
0146 1345: "Singularity: Failed to create user namespace",
0147 1346: "Transform not found",
0148 1347: "Unsupported SL5 OS",
0149 1348: "Singularity: Resource temporarily unavailable",
0150 1349: "Unrecognized transform arguments",
0151 1350: "Empty output file detected",
0152 1351: "Unrecognized fatal error in transform stderr",
0153 1352: "Failed to stat proc file for CPU consumption calculation",
0154 1353: "CPU consumption calculation failed: No such process",
0155 1354: "General CPU consumption calculation problem (consult Pilot log)",
0156 1355: "Core dump detected",
0157 1356: "Pre-process command failed",
0158 1357: "Post-process command failed",
0159 1358: "Missing release setup in unpacked container",
0160 1359: "PanDA queue is not active",
0161 1360: "Image not found",
0162 1361: "Remote file could not be opened",
0163 1362: "Xrdcp was unable to open file",
0164 1364: "Unable to locate credentials for S3 transfer",
0165 1365: "Python module ctypes not available on worker node",
0166 }
0167
0168
0169 def category_label(component, code):
0170 """Human label for an error category, e.g. 'pilot 1099 - Failed to stage-in file'."""
0171 try:
0172 code = int(code)
0173 except (TypeError, ValueError):
0174 return f"{component} {code}"
0175 message = None
0176 if component == "pilot":
0177 message = PILOT_LABELS.get(code)
0178 elif component == "taskbuffer":
0179 message = TASKBUFFER_LABELS.get(code)
0180 base = f"{component} {code}"
0181 return f"{base} - {message}" if message else base
