pilot/info/basedata.py

0001 # Licensed under the Apache License, Version 2.0 (the "License");
0002 # you may not use this file except in compliance with the License.
0003 # You may obtain a copy of the License at
0004 # http://www.apache.org/licenses/LICENSE-2.0
0005 #
0006 # Authors:
0007 # - Alexey Anisenkov, anisyonk@cern.ch, 2018
0008 # - Paul Nilsson, paul.nilsson@cern.ch, 2019
0009
0010 """
0011 The implementation of base data structure to host various settings collected
0012 from external source with built-in validation and schema translation support.
0013
0014 The main reasons for such incapsulation are to
0015  - apply in one place all data validation actions (for attributes and values)
0016  - introduce internal information schema (names of attribues) to remove dependency
0017  with data structrure, formats, names from external sources (e.g. AGIS/CRIC)
0018
0019
0020 :author: Alexey Anisenkov
0021 :contact: anisyonk@cern.ch
0022 :date: January 2018
0023 """
0024
0025 import ast
0026 import copy
0027 import logging
0028 logger = logging.getLogger(__name__)
0029
0030
0031 class BaseData(object):
0032     """
0033         High-level object to host structured data collected from external source
0034         It's considered to be like a bridge (connector) in order to remove direct dependency to
0035         external schema (format) implementation
0036     """
0037
0038     _keys = {}
0039
0040     def _load_data(self, data, kmap={}, validators=None):
0041         """
0042             Construct and initialize data from ext source.
0043
0044             :param data: input dictionary of raw data settings
0045             :param kmap: the translation map of data attributes from external format to internal schema
0046             :param validators: map of validation handlers to be applied
0047         """
0048
0049         # the translation map of the queue data attributes from external data to internal schema
0050         # 'internal_name':('ext_name1', 'extname2_if_any')
0051         # 'internal_name2':'ext_name3'
0052
0053         # first defined ext field will be used
0054         # if key is not explicitly specified then ext name will be used as is
0055         ## fix me later to proper internal names if need
0056
0057         #kmap = {
0058         #    # 'internal_name':('ext_name1', 'extname2_if_any'),
0059         #    # 'internal_name2':'ext_name3'
0060         #    }
0061
0062         if validators is None:
0063             # default validators
0064             validators = {int: self.clean_numeric,
0065                           str: self.clean_string,
0066                           bool: self.clean_boolean,
0067                           dict: self.clean_dictdata,
0068
0069                           None: self.clean_string,  # default validator
0070                           }
0071
0072         try:
0073             _items = list(self._keys.items())  # Python 3
0074         except Exception:
0075             _items = self._keys.iteritems()  # Python 2
0076         for ktype, knames in _items:
0077
0078             for kname in knames:
0079                 raw, value = None, None
0080
0081                 ext_names = kmap.get(kname) or kname
0082                 try:
0083                     if isinstance(ext_names, basestring):  # Python 2  # noqa: F821
0084                         ext_names = [ext_names]
0085                 except Exception:
0086                     if isinstance(ext_names, str):  # Python 3
0087                         ext_names = [ext_names]
0088
0089                 for name in ext_names:
0090                     raw = data.get(name)
0091                     if raw is not None:
0092                         break
0093
0094                 ## cast to required type and apply default validation
0095                 hvalidator = validators.get(ktype, validators.get(None))
0096                 if callable(hvalidator):
0097                     value = hvalidator(raw, ktype, kname, defval=copy.deepcopy(getattr(self, kname, None)))
0098                 ## apply custom validation if defined
0099                 hvalidator = getattr(self, 'clean__%s' % kname, None)
0100                 if callable(hvalidator):
0101                     value = hvalidator(raw, value)
0102
0103                 setattr(self, kname, value)
0104
0105         self.clean()
0106
0107     def clean(self):
0108         """
0109             Validate and finally clean up required data values (required object properties) if need
0110             Executed once all fields have already passed field-specific validation checks
0111             Could be customized by child object
0112             :return: None
0113         """
0114         pass
0115
0116     ##
0117     ## default validators
0118     ##
0119     def clean_numeric(self, raw, ktype, kname=None, defval=0):
0120         """
0121             Clean and convert input value to requested numeric type
0122             :param raw: raw input data
0123             :param ktype: variable type to which result should be casted
0124             :param defval: default value to be used in case of cast error
0125         """
0126
0127         if isinstance(raw, ktype):
0128             return raw
0129
0130         try:
0131             if isinstance(raw, basestring):  # Python 2  # noqa: F821
0132                 raw = raw.strip()
0133         except Exception:
0134             if isinstance(raw, str):  # Python 3
0135                 raw = raw.strip()
0136
0137         try:
0138             return ktype(raw)
0139         except Exception:
0140             if raw is not None:
0141                 logger.warning('failed to convert data for key=%s, raw=%s to type=%s, defval=%s' % (kname, raw, ktype, defval))
0142             return defval
0143
0144     def clean_string(self, raw, ktype, kname=None, defval=""):
0145         """
0146             Clean and convert input value to requested string type
0147             :param raw: raw input data
0148             :param ktype: variable type to which result should be casted
0149             :param defval: default value to be used in case of cast error
0150         """
0151
0152         if isinstance(raw, ktype):
0153             return raw
0154
0155         if raw is None:
0156             return defval
0157         else:
0158             try:
0159                 if isinstance(raw, basestring):  # Python 2  # noqa: F821
0160                     raw = raw.strip()
0161             except Exception:
0162                 if isinstance(raw, str):  # Python 3
0163                     raw = raw.strip()
0164         try:
0165             return ktype(raw)
0166         except Exception:
0167             logger.warning('failed to convert data for key=%s, raw=%s to type=%s' % (kname, raw, ktype))
0168             return defval
0169
0170     def clean_boolean(self, raw, ktype, kname=None, defval=None):
0171         """
0172             Clean and convert input value to requested boolean type
0173             :param raw: raw input data
0174             :param ktype: variable type to which result should be casted
0175             :param defval: default value to be used in case of cast error
0176         """
0177
0178         if isinstance(raw, ktype):
0179             return raw
0180
0181         if raw is None:  ## not set value, use default
0182             return defval
0183
0184         val = str(raw).strip().lower()
0185         allowed_values = ['', 'none', 'true', 'false', 'yes', 'no', '1', '0']
0186
0187         if val not in allowed_values:
0188             logger.warning('failed to convert data for key=%s, raw=%s to type=%s' % (kname, raw, ktype))
0189             return defval
0190
0191         return val.lower() in ['1', 'true', 'yes']
0192
0193     def clean_dictdata(self, raw, ktype, kname=None, defval=None):
0194         """
0195             Clean and convert input value to requested dict type
0196             :param raw: raw input data
0197             :param ktype: variable type to which result should be casted
0198             :param defval: default value to be used in case of cast error
0199         """
0200
0201         if isinstance(raw, str):
0202             raw = ast.literal_eval(raw)
0203
0204         if isinstance(raw, ktype):
0205             return raw
0206
0207         elif raw is None:
0208             return defval
0209         try:
0210             return ktype(raw)
0211         except Exception:
0212             logger.warning('failed to convert data for key=%s, raw=%s to type=%s' % (kname, raw, ktype))
0213             return defval
0214
0215     def clean_listdata(self, raw, ktype, kname=None, defval=None):
0216         """
0217             Clean and convert input value to requested list type
0218             :param raw: raw input data
0219             :param ktype: variable type to which result should be casted
0220             :param defval: default value to be used in case of cast error
0221         """
0222
0223         if isinstance(raw, ktype):
0224             return raw
0225
0226         elif raw is None:
0227             return defval
0228         else:
0229             try:
0230                 if isinstance(raw, basestring):  # Python 2  # noqa: F821
0231                     raw = raw.split(',')
0232             except Exception:
0233                 if isinstance(raw, str):  # Python 3
0234                     raw = raw.split(',')
0235         try:
0236             return ktype(raw)
0237         except Exception:
0238             logger.warning('failed to convert data for key=%s, raw=%s to type=%s' % (kname, raw, ktype))
0239             return defval
0240
0241     ## custom function pattern to apply extra validation to the give key values
0242     #def clean__keyname(self, raw, value):
0243     #  :param raw: raw value passed from ext source as input
0244     #  :param value: preliminary cleaned and casted to proper type value
0245     #
0246     #    return value
0247
0248     def __repr__(self):
0249         """
0250             Default representation of an object
0251         """
0252
0253         ret = []
0254         attrs = [key for key in dir(self) if not callable(getattr(self, key)) and not key.startswith('_')]
0255         for key in sorted(attrs):
0256             ret.append(" %s=%s" % (key, getattr(self, key)))
0257         ret.append('')
0258         return '\n'.join(ret)