pilot/info/storagedata.py

0001 # Licensed under the Apache License, Version 2.0 (the "License");
0002 # you may not use this file except in compliance with the License.
0003 # You may obtain a copy of the License at
0004 # http://www.apache.org/licenses/LICENSE-2.0
0005 #
0006 # Authors:
0007 # - Alexey Anisenkov, anisyonk@cern.ch, 2018
0008 # - Paul Nilsson, paul.nilsson@cern.ch, 2019-21
0009
0010 """
0011 The implementation of data structure to host storage data description.
0012
0013 The main reasons for such incapsulation are to
0014  - apply in one place all data validation actions (for attributes and values)
0015  - introduce internal information schema (names of attribues) to remove direct dependency
0016  with data structrure, formats, names from external sources (e.g. AGIS/CRIC)
0017
0018 :author: Alexey Anisenkov
0019 :contact: anisyonk@cern.ch
0020 :date: January 2018
0021 """
0022 import traceback
0023 from os import environ
0024
0025 from pilot.util import https
0026 from pilot.util.config import config
0027 from .basedata import BaseData
0028
0029 import logging
0030 logger = logging.getLogger(__name__)
0031
0032
0033 class StorageData(BaseData):
0034     """
0035         High-level object to host Storage details (available protocols, etc.)
0036     """
0037
0038     ## put explicit list of all the attributes with comments for better inline-documentation by sphinx
0039     ## FIX ME LATER: use proper doc format
0040
0041     ## incomplete list of attributes .. to be extended once becomes used
0042
0043     pk = 0        # unique identification number
0044     name = ""     # DDMEndpoint name
0045     type = ""     # type of Storage <- can this be renamed to storagetype without causing any problem with queuedata?
0046     token = ""    # space token descriptor
0047
0048     is_deterministic = None
0049
0050     state = None
0051     site = None   # ATLAS Site name
0052
0053     arprotocols = {}
0054     rprotocols = {}
0055     special_setup = {}
0056     resource = None
0057
0058     # specify the type of attributes for proper data validation and casting
0059     _keys = {int: ['pk'],
0060              str: ['name', 'state', 'site', 'type', 'token'],
0061              dict: ['copytools', 'acopytools', 'astorages', 'arprotocols', 'rprotocols', 'resource'],
0062              bool: ['is_deterministic']
0063              }
0064
0065     def __init__(self, data):
0066         """
0067             :param data: input dictionary of storage description by DDMEndpoint name as key
0068         """
0069
0070         self.load(data)
0071
0072         # DEBUG
0073         #import pprint
0074         #logger.debug('initialize StorageData from raw:\n%s' % pprint.pformat(data))
0075         #logger.debug('Final parsed StorageData content:\n%s' % self)
0076
0077     def load(self, data):
0078         """
0079             Construct and initialize data from ext source
0080             :param data: input dictionary of storage description by DDMEndpoint name as key
0081         """
0082
0083         # the translation map of the queue data attributes from external data to internal schema
0084         # first defined ext field name will be used
0085         # if key is not explicitly specified then ext name will be used as is
0086         ## fix me later to proper internal names if need
0087
0088         kmap = {
0089             # 'internal_name': ('ext_name1', 'extname2_if_any')
0090             # 'internal_name2': 'ext_name3'
0091             'pk': 'id',
0092         }
0093
0094         self._load_data(data, kmap)
0095
0096     ## custom function pattern to apply extra validation to the key values
0097     ##def clean__keyname(self, raw, value):
0098     ##  :param raw: raw value passed from ext source as input
0099     ##  :param value: preliminary cleaned and casted to proper type value
0100     ##
0101     ##    return value
0102
0103     # to be improved: move it to some data loader
0104     def get_security_key(self, secret_key, access_key):
0105         """
0106             Get security key pair from panda
0107             :param secret_key: secrect key name as string
0108             :param access_key: access key name as string
0109             :return: setup as a string
0110         """
0111         try:
0112             data = {'privateKeyName': secret_key, 'publicKeyName': access_key}
0113             logger.info("Getting key pair: %s" % data)
0114             url = environ.get('PANDA_SERVER_URL', config.Pilot.pandaserver)
0115             res = https.request('{pandaserver}/server/panda/getKeyPair'.format(pandaserver=url), data=data)
0116             if res and res['StatusCode'] == 0:
0117                 return {"publicKey": res["publicKey"], "privateKey": res["privateKey"]}
0118             else:
0119                 logger.info("Got key pair returns wrong value: %s" % res)
0120         except Exception as ex:
0121             logger.error("Failed to get key pair(%s,%s): %s, %s" % (access_key, secret_key, ex, traceback.format_exc()))
0122         return {}
0123
0124     def get_special_setup(self, protocol_id=None):
0125         """
0126         Construct special setup for ddms such as objectstore
0127         :param protocol_id: protocol id.
0128         :return: setup as a string
0129         """
0130
0131         logger.info("Get special setup for protocol id(%s)" % (protocol_id))
0132         if protocol_id in self.special_setup and self.special_setup[protocol_id]:
0133             return self.special_setup[protocol_id]
0134
0135         if protocol_id is None or str(protocol_id) not in list(self.rprotocols.keys()):  # Python 2/3
0136             return None
0137
0138         if self.type in ['OS_ES', 'OS_LOGS']:
0139             self.special_setup[protocol_id] = None
0140
0141             settings = self.rprotocols.get(str(protocol_id), {}).get('settings', {})
0142             access_key = settings.get('access_key', None)
0143             secret_key = settings.get('secret_key', None)
0144             is_secure = settings.get('is_secure', None)
0145
0146             # make sure all things are correctly defined in AGIS.
0147             # If one of them is not defined correctly, will not setup this part. Then rucio client can try to use signed url.
0148             # This part is preferred because signed url is not efficient.
0149             if access_key and secret_key and is_secure:
0150                 key_pair = self.get_security_key(secret_key, access_key)
0151                 if "privateKey" not in key_pair or key_pair["privateKey"] is None:
0152                     logger.error("Failed to get the key pair for S3 objectstore from panda")
0153                 else:
0154                     setup = "export S3_ACCESS_KEY=%s; export S3_SECRET_KEY=%s; export S3_IS_SECURE=%s;" % (key_pair["publicKey"],
0155                                                                                                            key_pair["privateKey"],
0156                                                                                                            is_secure)
0157                     self.special_setup[protocol_id] = setup
0158                     logger.info("Return key pair with public key: %s" % key_pair["publicKey"])
0159                     return self.special_setup[protocol_id]
0160         return None