pilot/workflow/eventservice_hpc.py

0001 #!/usr/bin/env python
0002 # Licensed under the Apache License, Version 2.0 (the "License");
0003 # you may not use this file except in compliance with the License.
0004 # You may obtain a copy of the License at
0005 # http://www.apache.org/licenses/LICENSE-2.0
0006 #
0007 # Authors:
0008 # - Mario Lassnig, mario.lassnig@cern.ch, 2016
0009 # - Paul Nilsson, paul.nilsson@cern.ch, 2018-2019
0010
0011 import functools
0012 import signal
0013 from collections import namedtuple
0014 from os import environ
0015
0016 from pilot.util.constants import SUCCESS, FAILURE
0017
0018 import logging
0019 logger = logging.getLogger(__name__)
0020
0021
0022 def interrupt(args, signum, frame):
0023     try:
0024         logger.info('caught signal: %s' % [v for v, k in signal.__dict__.iteritems() if k == signum][0])
0025     except Exception:
0026         logger.info('caught signal: %s' % [v for v, k in list(signal.__dict__.items()) if k == signum][0])
0027     args.graceful_stop.set()
0028
0029
0030 def run(args):
0031     """
0032     Main execution function for the event service workflow on HPCs (Yoda-Droid).
0033
0034     :param args: pilot arguments.
0035     :returns: traces object.
0036     """
0037
0038     try:
0039         logger.info('setting up signal handling')
0040         signal.signal(signal.SIGINT, functools.partial(interrupt, args))
0041
0042         logger.info('setting up tracing')
0043         traces = namedtuple('traces', ['pilot'])
0044         traces.pilot = {'state': SUCCESS,
0045                         'nr_jobs': 0}
0046
0047         if args.hpc_resource == '':
0048             logger.critical('hpc resource not specified, cannot continue')
0049             traces.pilot['state'] = FAILURE
0050             return traces
0051
0052         # get the resource reference
0053         resource = __import__('pilot.resource.%s' % args.hpc_resource, globals(), locals(), [args.hpc_resource], 0)  # Python 2/3
0054
0055         # example usage:
0056         logger.info('setup for resource %s: %s' % (args.hpc_resource, str(resource.get_setup())))
0057
0058         # are we Yoda or Droid?
0059         if environ.get('SOME_ENV_VARIABLE', '') == 'YODA':
0060             yodadroid = __import__('pilot.eventservice.yoda')
0061         else:
0062             yodadroid = __import__('pilot.eventservice.droid')
0063         yodadroid.run()
0064
0065     except Exception as e:
0066         logger.fatal('exception caught: %s' % e)
0067
0068     return traces