pandaharvester/harvesterconfig/harvester_config.py

0001 import json
0002 import os
0003 import re
0004 import socket
0005 import sys
0006
0007 from liveconfigparser.LiveConfigParser import LiveConfigParser
0008
0009 # get ConfigParser
0010 tmpConf = LiveConfigParser()
0011
0012 # URL for config file if any
0013 configEnv = "HARVESTER_INSTANCE_CONFIG_URL"
0014 if configEnv in os.environ:
0015     configURL = os.environ[configEnv]
0016 else:
0017     configURL = None
0018
0019 # read
0020 tmpConf.read("panda_harvester.cfg", configURL)
0021
0022
0023 # get the value of env var in the config
0024 def env_var_parse(val):
0025     match = re.search("\$\{*([^\}]+)\}*", val)
0026     if match is None:
0027         return val
0028     var_name = match.group(1)
0029     if var_name.upper() == "HOSTNAME":
0030         return socket.gethostname().split(".")[0]
0031     if var_name not in os.environ:
0032         raise KeyError(f"{var_name} in the cfg is an undefined environment variable.")
0033     else:
0034         return os.environ[var_name]
0035
0036
0037 # env var substitution in all values in nested list + dict which parsed from json object
0038 def nested_obj_env_var_sub(obj):
0039     if isinstance(obj, list):
0040         for i, v in enumerate(obj):
0041             if isinstance(v, str):
0042                 obj[i] = env_var_parse(v)
0043             else:
0044                 nested_obj_env_var_sub(v)
0045     elif isinstance(obj, dict):
0046         for k, v in obj.items():
0047             if isinstance(v, str):
0048                 obj[k] = env_var_parse(v)
0049             else:
0050                 nested_obj_env_var_sub(v)
0051
0052
0053 # dummy section class
0054 class _SectionClass:
0055     def __init__(self):
0056         pass
0057
0058
0059 # load configmap
0060 config_map_data = {}
0061 if "PANDA_HOME" in os.environ:
0062     config_map_name = "panda_harvester_configmap.json"
0063     config_map_path = os.path.join(os.environ["PANDA_HOME"], "etc/configmap", config_map_name)
0064     if os.path.exists(config_map_path):
0065         with open(config_map_path) as f:
0066             config_map_data = json.load(f)
0067
0068
0069 # config format in dict for print only
0070 config_dict = {}
0071
0072 # loop over all sections
0073 for tmpSection in tmpConf.sections():
0074     # read section
0075     tmpDict = getattr(tmpConf, tmpSection)
0076     # load configmap
0077     if tmpSection in config_map_data:
0078         tmpDict.update(config_map_data[tmpSection])
0079     # make section class
0080     tmpSelf = _SectionClass()
0081     # update module dict
0082     sys.modules[__name__].__dict__[tmpSection] = tmpSelf
0083     # initialize config dict
0084     config_dict[tmpSection] = {}
0085     # expand all values
0086     for tmpKey, tmpVal in tmpDict.items():
0087         # use env vars
0088         if isinstance(tmpVal, str) and tmpVal.startswith("$"):
0089             tmpVal = env_var_parse(tmpVal)
0090         # convert string to bool/int
0091         if not isinstance(tmpVal, str):
0092             pass
0093         elif tmpVal == "True":
0094             tmpVal = True
0095         elif tmpVal == "False":
0096             tmpVal = False
0097         elif tmpVal == "None":
0098             tmpVal = None
0099         elif re.match("^\d+$", tmpVal):
0100             tmpVal = int(tmpVal)
0101         elif "\n" in tmpVal and (re.match(r"^\W*\[.*\]\W*$", tmpVal.replace("\n", "")) or re.match(r"^\W*\{.*\}\W*$", tmpVal.replace("\n", ""))):
0102             tmpVal = json.loads(tmpVal)
0103             nested_obj_env_var_sub(tmpVal)
0104         elif "\n" in tmpVal:
0105             tmpVal = tmpVal.split("\n")
0106             # remove empty
0107             tmpVal = [x.strip() for x in tmpVal if x.strip()]
0108         # update dict
0109         setattr(tmpSelf, tmpKey, tmpVal)
0110         # update config dict
0111         if not any(ss in tmpKey.lower() for ss in ["password", "passphrase", "secret"]):
0112             config_dict[tmpSection][tmpKey] = tmpVal