pandaserver/config/panda_config.py

0001 import glob
0002 import json
0003 import os
0004 import re
0005 import socket
0006 import sys
0007
0008 from pandacommon.liveconfigparser.LiveConfigParser import (
0009     LiveConfigParser,
0010     expand_values,
0011 )
0012
0013 from . import config_utils
0014
0015 # get ConfigParser
0016 tmpConf = LiveConfigParser()
0017
0018 # read
0019 tmpConf.read("panda_server.cfg")
0020
0021 # get server section
0022 tmpDict = tmpConf.server
0023
0024 # read configmap
0025 config_utils.load_config_map("server", tmpDict)
0026
0027 # expand all values
0028 tmpSelf = sys.modules[__name__]
0029 expand_values(tmpSelf, tmpDict)
0030
0031 # set hostname
0032 if "pserverhost" not in tmpSelf.__dict__:
0033     tmpSelf.__dict__["pserverhost"] = socket.getfqdn()
0034
0035 # set port for http
0036 if "pserverporthttp" not in tmpSelf.__dict__:
0037     tmpSelf.__dict__["pserverporthttp"] = 25080
0038
0039 # set host for http
0040 if "pserverhosthttp" not in tmpSelf.__dict__:
0041     tmpSelf.__dict__["pserverhosthttp"] = tmpSelf.__dict__["pserverhost"]
0042
0043 # disable http
0044 if "disableHTTP" not in tmpSelf.__dict__:
0045     tmpSelf.__dict__["disableHTTP"] = False
0046
0047 # DB backend
0048 if "backend" not in tmpSelf.__dict__:
0049     tmpSelf.__dict__["backend"] = "oracle"
0050 if "dbport" not in tmpSelf.__dict__:
0051     tmpSelf.__dict__["dbport"] = 0
0052 if "dbtimeout" not in tmpSelf.__dict__:
0053     tmpSelf.__dict__["dbtimeout"] = 60
0054
0055 # Directory for certs
0056 if "certdir" not in tmpSelf.__dict__:
0057     tmpSelf.__dict__["certdir"] = "/data/atlpan"
0058
0059 # endpoint map file
0060 if "endpoint_mapfile" not in tmpSelf.__dict__:
0061     tmpSelf.__dict__["endpoint_mapfile"] = (
0062         "/cvmfs/atlas.cern.ch/repo/sw/local/etc/cric_ddmendpoints.json" ",/cvmfs/atlas.cern.ch/repo/sw/local/etc/agis_ddmendpoints.json"
0063     )
0064
0065 # sandbox info
0066 if "record_sandbox_info" not in tmpSelf.__dict__:
0067     tmpSelf.__dict__["record_sandbox_info"] = True
0068
0069 # secrets
0070 if "pilot_secrets" not in tmpSelf.__dict__:
0071     tmpSelf.__dict__["pilot_secrets"] = "pilot secrets"
0072
0073 # schemas
0074 if "schemaPANDA" not in tmpSelf.__dict__:
0075     tmpSelf.__dict__["schemaPANDA"] = "ATLAS_PANDA"
0076 if "schemaPANDAARCH" not in tmpSelf.__dict__:
0077     tmpSelf.__dict__["schemaPANDAARCH"] = "ATLAS_PANDAARCH"
0078 if "schemaMETA" not in tmpSelf.__dict__:
0079     tmpSelf.__dict__["schemaMETA"] = "ATLAS_PANDAMETA"
0080 if "schemaJEDI" not in tmpSelf.__dict__:
0081     tmpSelf.__dict__["schemaJEDI"] = "ATLAS_PANDA"
0082 if "schemaDEFT" not in tmpSelf.__dict__:
0083     tmpSelf.__dict__["schemaDEFT"] = "ATLAS_DEFT"
0084 if "schemaGRISLI" not in tmpSelf.__dict__:
0085     tmpSelf.__dict__["schemaGRISLI"] = "ATLAS_GRISLI"
0086 if "schemaEI" not in tmpSelf.__dict__:
0087     tmpSelf.__dict__["schemaEI"] = "ATLAS_EVENTINDEX"
0088
0089 # default site
0090 if "def_sitename" not in tmpSelf.__dict__:
0091     tmpSelf.__dict__["def_sitename"] = "BNL_ATLAS_1"
0092 if "def_queue" not in tmpSelf.__dict__:
0093     tmpSelf.__dict__["def_queue"] = "ANALY_BNL_ATLAS_1"
0094 if "def_nickname" not in tmpSelf.__dict__:
0095     tmpSelf.__dict__["def_nickname"] = "BNL_ATLAS_1-condor"
0096 if "def_ddm" not in tmpSelf.__dict__:
0097     tmpSelf.__dict__["def_ddm"] = "PANDA_UNDEFINED2"
0098 if "def_type" not in tmpSelf.__dict__:
0099     tmpSelf.__dict__["def_type"] = "production"
0100 if "def_status" not in tmpSelf.__dict__:
0101     tmpSelf.__dict__["def_status"] = "online"
0102 if "token_authType" not in tmpSelf.__dict__:
0103     tmpSelf.__dict__["token_authType"] = "oidc"
0104 if "auth_config" not in tmpSelf.__dict__:
0105     tmpSelf.__dict__["auth_config"] = "/opt/panda/etc/panda/auth/"
0106 if "token_audience" not in tmpSelf.__dict__:
0107     tmpSelf.__dict__["token_audience"] = "https://pandaserver.cern.ch"
0108 if "token_issuers" not in tmpSelf.__dict__:
0109     tmpSelf.__dict__["token_issuers"] = ""
0110 if "legacy_token_issuers" not in tmpSelf.__dict__:
0111     tmpSelf.__dict__["legacy_token_issuers"] = "https://atlas-auth.cern.ch/,https://atlas-auth.web.cern.ch/"
0112 if tmpSelf.__dict__["legacy_token_issuers"]:
0113     tmpSelf.__dict__["legacy_token_issuers"] = tmpSelf.__dict__["legacy_token_issuers"].split(",")
0114 else:
0115     tmpSelf.__dict__["legacy_token_issuers"] = None
0116 tmpSelf.__dict__["production_dns"] = [x for x in tmpSelf.__dict__.get("production_dns", "").split(",") if x]
0117 tmpSelf.__dict__["pilot_owners"] = [x for x in tmpSelf.__dict__.get("pilot_owners", "").split(",") if x]
0118 tmpSelf.__dict__["auth_policies"] = {}
0119 tmpSelf.__dict__["auth_vo_dict"] = {}
0120 try:
0121     data_dict = {}
0122     vo_data_dict = {}
0123     policy_dict = {}
0124     for name in glob.glob(os.path.join(tmpSelf.__dict__["auth_config"], "*_auth_config.json")):
0125         with open(name) as f:
0126             data = json.load(f)
0127             data_dict[data["client_id"]] = data
0128             if "secondary_ids" in data:
0129                 for tmp_id in data["secondary_ids"]:
0130                     data_dict[tmp_id] = data
0131         m = re.search("^(.+)_auth_config.json", os.path.basename(name))
0132         if m:
0133             # Extract the VO group from the filename
0134             tmp_vo_group = m.group(1)
0135
0136             # Determine tmp_vo and tmp_group based on delimiters
0137             if tmp_vo_group.startswith("vo.") and ":" in tmp_vo_group:
0138                 tmp_vo, tmp_group = tmp_vo_group.split(":", 1)
0139             elif ":" in tmp_vo_group:
0140                 tmp_vo, tmp_group = tmp_vo_group.split(":", 1)
0141             elif tmp_vo_group.startswith("vo."):
0142                 # If it starts with "vo." but has no ":", treat it as a single VO with "user" as the group
0143                 tmp_vo = tmp_vo_group
0144                 tmp_group = "user"
0145             elif "." in tmp_vo_group:
0146                 tmp_vo, tmp_group = tmp_vo_group.split(".", 1)
0147             else:
0148                 tmp_vo, tmp_group = tmp_vo_group, "user"
0149
0150             policy_dict.setdefault(tmp_vo, [])
0151             policy_dict[tmp_vo].append([tmp_vo, {"group": tmp_group, "role": tmp_group}])
0152             vo_data_dict[tmp_vo_group.replace(":", ".")] = data
0153     tmpSelf.__dict__["auth_config"] = data_dict
0154     tmpSelf.__dict__["auth_policies"] = policy_dict
0155     tmpSelf.__dict__["auth_vo_dict"] = vo_data_dict
0156 except Exception:
0157     tmpSelf.__dict__["auth_config"] = {}
0158
0159 if "token_cache_config" not in tmpSelf.__dict__:
0160     tmpSelf.__dict__["token_cache_config"] = "/opt/panda/etc/panda/token_cache_config.json"
0161
0162 # use cert in configurator
0163 if "configurator_use_cert" not in tmpSelf.__dict__:
0164     tmpSelf.__dict__["configurator_use_cert"] = True
0165
0166 # adder serialization
0167 if "add_serialized" not in tmpSelf.__dict__:
0168     tmpSelf.__dict__["add_serialized"] = False
0169
0170 # disable file aggregation
0171 if "disable_file_aggregation" not in tmpSelf.__dict__:
0172     tmpSelf.__dict__["disable_file_aggregation"] = False
0173
0174 # dict for plugins
0175 g_pluginMap = {}
0176
0177
0178 # parser for plugin setup
0179 def parsePluginConf(modConfigName):
0180     global tmpSelf
0181     global g_pluginMap
0182     g_pluginMap.setdefault(modConfigName, {})
0183     # parse plugin setup
0184     try:
0185         for configStr in getattr(tmpSelf, modConfigName).split(","):
0186             configStr = configStr.strip()
0187             items = configStr.split(":")
0188             vos = items[0].split("|")
0189             moduleName = items[1]
0190             className = items[2]
0191             if len(items) > 3:
0192                 group = items[3]
0193             else:
0194                 group = None
0195             for vo in vos:
0196                 # import
0197                 mod = __import__(moduleName)
0198                 for subModuleName in moduleName.split(".")[1:]:
0199                     mod = getattr(mod, subModuleName)
0200                 # get class
0201                 cls = getattr(mod, className)
0202                 if group:
0203                     vo_key = f"{vo}_{group}"
0204                 else:
0205                     vo_key = vo
0206                 g_pluginMap[modConfigName][vo_key] = cls
0207     except Exception:
0208         pass
0209
0210
0211 # accessor for plugin
0212 def getPlugin(modConfigName, vo, group=None):
0213     if modConfigName not in g_pluginMap:
0214         return None
0215     if group:
0216         vo_group = f"{vo}_{group}"
0217         if vo_group in g_pluginMap[modConfigName]:
0218             # VO+group specified
0219             return g_pluginMap[modConfigName][vo_group]
0220     if vo in g_pluginMap[modConfigName]:
0221         # VO specified
0222         return g_pluginMap[modConfigName][vo]
0223     if "any" in g_pluginMap[modConfigName]:
0224         # catch all
0225         return g_pluginMap[modConfigName]["any"]
0226     # undefined
0227     return None
0228
0229
0230 # plug-ins
0231 def setupPlugin():
0232     parsePluginConf("adder_plugins")
0233     parsePluginConf("setupper_plugins")