api/v1/http_client.py

0001 """
0002 client methods
0003 """
0004
0005 import os
0006
0007 import requests
0008 from pandacommon.pandautils.net_utils import replace_hostname_in_url_randomly
0009
0010 # PanDA server configuration
0011 base_url = os.environ.get("PANDA_URL", "http://pandaserver.cern.ch:25080/server/panda")
0012 base_url_ssl = os.environ.get("PANDA_URL_SSL", "https://pandaserver.cern.ch:25443/server/panda")
0013
0014 api_url = os.environ.get("PANDA_API_URL", "http://pandaserver.cern.ch:25080/api/v1")
0015 api_url_ssl = os.environ.get("PANDA_API_URL_SSL", "https://pandaserver.cern.ch:25443/api/v1")
0016
0017 # exit code
0018 EC_Failed = 255
0019
0020
0021 def is_https(url):
0022     # check if https is used
0023     return url.startswith("https://")
0024
0025
0026 class HttpClient:
0027     def __init__(self):
0028         # verification of the host certificate
0029         if "PANDA_VERIFY_HOST" in os.environ and os.environ["PANDA_VERIFY_HOST"] == "off":
0030             self.verifyHost = False
0031         else:
0032             self.verifyHost = True
0033
0034         # request a compressed response
0035         self.compress = True
0036
0037         # SSL cert/key
0038         self.ssl_certificate = self._x509()
0039         self.ssl_key = self._x509()
0040
0041         # OIDC
0042         self.oidc = os.getenv("PANDA_AUTH") == "oidc"
0043         self.auth_vo = os.getenv("PANDA_AUTH_VO") if self.oidc else None
0044         self.id_token = os.getenv("PANDA_AUTH_ID_TOKEN") if self.oidc else None
0045         if self.id_token and self.id_token.startswith("file:"):
0046             with open(self.id_token[5:], "r") as f:
0047                 self.id_token = f.read().strip()
0048
0049     def _x509(self):
0050         # retrieve the X509_USER_PROXY from the environment variables
0051         try:
0052             return os.environ["X509_USER_PROXY"]
0053         except Exception:
0054             pass
0055
0056         # look for the default place
0057         x509 = f"/tmp/x509up_u{os.getuid()}"
0058         if os.access(x509, os.R_OK):
0059             return x509
0060
0061         # no valid proxy certificate
0062         print("No valid grid proxy certificate found")
0063         return ""
0064
0065     def _prepare_url(self, url):
0066         """Modify URL with HTTPS check and hostname replacement."""
0067         use_https = is_https(url)
0068         if "PANDA_BEHIND_REAL_LB" in os.environ:
0069             modified_url = url
0070         else:
0071             modified_url = replace_hostname_in_url_randomly(url)
0072         return modified_url, use_https
0073
0074     def _prepare_headers(self, accept_json=True, content_type_json=True, encoding=None):
0075         """Prepare headers based on authentication and JSON settings."""
0076         headers = {}
0077         if accept_json:
0078             headers["Accept"] = "application/json"
0079         if content_type_json:
0080             headers["Content-Type"] = "application/json"
0081
0082         if encoding:
0083             headers["Content-Encoding"] = encoding
0084
0085         if self.oidc:
0086             headers["Authorization"] = f"Bearer {self.id_token}"
0087             headers["Origin"] = self.auth_vo
0088
0089         return headers
0090
0091     def _prepare_ssl(self, use_https):
0092         """Prepare SSL configuration based on HTTPS usage and verification settings."""
0093         cert = None
0094         verify = True
0095         if use_https:
0096             if not self.oidc:
0097                 cert = (self.ssl_certificate, self.ssl_key)
0098
0099             if not self.verifyHost:
0100                 verify = False
0101             elif "X509_CERT_DIR" in os.environ:
0102                 verify = os.environ["X509_CERT_DIR"]
0103             elif os.path.exists("/etc/grid-security/certificates"):
0104                 verify = "/etc/grid-security/certificates"
0105
0106         return cert, verify
0107
0108     def get(self, url, data):
0109         url, use_https = self._prepare_url(url)
0110         headers = self._prepare_headers()
0111         cert, verify = self._prepare_ssl(use_https)
0112
0113         try:
0114             response = requests.get(url, headers=headers, params=data, timeout=600, cert=cert, verify=verify)
0115             response.raise_for_status()
0116             return 0, response.json()
0117         except requests.RequestException as e:
0118             return 255, str(e)
0119
0120     def post(self, url, data):
0121         url, use_https = self._prepare_url(url)
0122         headers = self._prepare_headers()
0123         cert, verify = self._prepare_ssl(use_https)
0124
0125         try:
0126             response = requests.post(url, headers=headers, json=data, timeout=600, cert=cert, verify=verify)
0127             response.raise_for_status()
0128             return 0, response.json()
0129         except requests.RequestException as e:
0130             return 255, str(e)
0131
0132     def post_files(self, url, data, encoding=None):
0133         url, use_https = self._prepare_url(url)
0134         headers = self._prepare_headers(content_type_json=False, encoding=encoding)
0135         cert, verify = self._prepare_ssl(use_https)
0136
0137         files = {}
0138         try:
0139             for key, value in data.items():
0140                 if isinstance(data[key], str):
0141                     # we got a file to upload without specifying the destination name
0142                     files[key] = open(data[key], "rb")
0143                 else:
0144                     # we got a file to upload which specifies the destination name
0145                     files[key] = (data[key][0], open(data[key][1], "rb"))
0146             print(f"cert: {cert}, verify: {verify}")
0147             response = requests.post(url, headers=headers, files=files, timeout=600, cert=cert, verify=verify)
0148             response.raise_for_status()
0149             return 0, response.json()
0150         except requests.RequestException as e:
0151             return 255, str(e)
0152         finally:
0153             for file in files.values():
0154                 if isinstance(file, tuple):
0155                     file_handler = file[1]
0156                 else:
0157                     file_handler = file
0158                 file_handler.close()
0159
0160     def override_oidc(self, oidc, id_token, auth_vo):
0161         """
0162         Override OIDC settings.
0163         """
0164         self.oidc = oidc
0165         self.id_token = id_token
0166         self.auth_vo = auth_vo