user/rubin/diagnose.py

0001 #!/usr/bin/env python
0002 # Licensed under the Apache License, Version 2.0 (the "License");
0003 # you may not use this file except in compliance with the License.
0004 # You may obtain a copy of the License at
0005 # http://www.apache.org/licenses/LICENSE-2.0
0006 #
0007 # Authors:
0008 # - Paul Nilsson, paul.nilsson@cern.ch, 2021
0009 # - Tadashi Maeno, tadashi.maeno@cern.ch, 2020
0010
0011 import os
0012
0013 from pilot.util.config import config
0014 from pilot.util.filehandling import read_file, tail
0015
0016 import logging
0017 logger = logging.getLogger(__name__)
0018
0019
0020 def interpret(job):
0021     """
0022     Interpret the payload, look for specific errors in the stdout.
0023
0024     :param job: job object
0025     :return: exit code (payload) (int).
0026     """
0027
0028     stdout = os.path.join(job.workdir, config.Payload.payloadstdout)
0029     message = 'payload stdout dump\n'
0030     message += read_file(stdout)
0031     logger.debug(message)
0032     stderr = os.path.join(job.workdir, config.Payload.payloadstderr)
0033     message = 'payload stderr dump\n'
0034     message += read_file(stderr)
0035     logger.debug(message)
0036
0037     return 0
0038
0039
0040 def get_log_extracts(job, state):
0041     """
0042     Extract special warnings and other other info from special logs.
0043     This function also discovers if the payload had any outbound connections.
0044
0045     :param job: job object.
0046     :param state: job state (string).
0047     :return: log extracts (string).
0048     """
0049
0050     logger.info("building log extracts (sent to the server as \'pilotLog\')")
0051
0052     # for failed/holding jobs, add extracts from the pilot log file, but always add it to the pilot log itself
0053     extracts = ""
0054     _extracts = get_pilot_log_extracts(job)
0055     if _extracts != "":
0056         logger.warning('detected the following tail of warning/fatal messages in the pilot log:\n%s' % _extracts)
0057         if state == 'failed' or state == 'holding':
0058             extracts += _extracts
0059
0060     return extracts
0061
0062
0063 def get_pilot_log_extracts(job):
0064     """
0065     Get the extracts from the pilot log (warning/fatal messages, as well as tail of the log itself).
0066
0067     :param job: job object.
0068     :return: tail of pilot log (string).
0069     """
0070
0071     extracts = ""
0072
0073     path = os.path.join(job.workdir, config.Pilot.pilotlog)
0074     if os.path.exists(path):
0075         # get the last 20 lines of the pilot log in case it contains relevant error information
0076         _tail = tail(path, nlines=20)
0077         if _tail != "":
0078             if extracts != "":
0079                 extracts += "\n"
0080             extracts += "- Log from %s -\n" % config.Pilot.pilotlog
0081             extracts += _tail
0082     else:
0083         logger.warning('pilot log file does not exist: %s' % path)
0084
0085     return extracts