pilot/util/https.py

0001 #!/usr/bin/env python
0002 # Licensed under the Apache License, Version 2.0 (the "License");
0003 # you may not use this file except in compliance with the License.
0004 # You may obtain a copy of the License at
0005 # http://www.apache.org/licenses/LICENSE-2.0
0006 #
0007 # Authors:
0008 # - Daniel Drizhuk, d.drizhuk@gmail.com, 2017
0009 # - Mario Lassnig, mario.lassnig@cern.ch, 2017
0010 # - Paul Nilsson, paul.nilsson@cern.ch, 2017-2020
0011
0012 import collections
0013 import subprocess  # Python 2/3
0014 try:
0015     import commands  # Python 2
0016 except Exception:
0017     pass
0018 import json
0019 import os
0020 import platform
0021 import ssl
0022 import sys
0023 try:
0024     import urllib.request  # Python 3
0025     import urllib.error  # Python 3
0026     import urllib.parse  # Python 3
0027 except Exception:
0028     import urllib  # Python 2
0029     import urllib2  # Python 2
0030 import pipes
0031
0032 from .filehandling import write_file
0033 from .auxiliary import is_python3
0034 from .config import config
0035 from .constants import get_pilot_version
0036
0037 import logging
0038 logger = logging.getLogger(__name__)
0039
0040 _ctx = collections.namedtuple('_ctx', 'ssl_context user_agent capath cacert')
0041
0042 # anisyonk: public copy of `_ctx` to avoid logic break since ssl_context is reset inside the request() -- FIXME
0043 # anisyonk: public instance, should be properly initialized by `https_setup()`
0044 # anisyonk: use lightweight class definition instead of namedtuple since tuple is immutable and we don't need/use any tuple features here
0045 ctx = type('ctx', (object,), dict(ssl_context=None, user_agent='Pilot2 client', capath=None, cacert=None))
0046
0047
0048 def _tester(func, *args):
0049     """
0050     Tests function ``func`` on arguments and returns first positive.
0051
0052     >>> _tester(lambda x: x%3 == 0, 1, 2, 3, 4, 5, 6)
0053     3
0054     >>> _tester(lambda x: x%3 == 0, 1, 2)
0055     None
0056
0057     :param func: function(arg)->boolean
0058     :param args: other arguments
0059     :return: something or none
0060     """
0061     for arg in args:
0062         if arg is not None and func(arg):
0063             return arg
0064
0065     return None
0066
0067
0068 def capath(args=None):
0069     """
0070     Tries to get :abbr:`CA (Certification Authority)` path with certificates.
0071     Testifies it to be a directory.
0072     Tries next locations:
0073
0074     1. :option:`--capath` from arguments
0075     2. :envvar:`X509_CERT_DIR` from env
0076     3. Path ``/etc/grid-security/certificates``
0077
0078     :param args: arguments, parsed by `argparse`
0079     :returns: `str` -- directory path, or `None`
0080     """
0081
0082     return _tester(os.path.isdir,
0083                    args and args.capath,
0084                    os.environ.get('X509_CERT_DIR'),
0085                    '/etc/grid-security/certificates')
0086
0087
0088 def cacert_default_location():
0089     """
0090     Tries to get current user ID through `os.getuid`, and get the posix path for x509 certificate.
0091     :returns: `str` -- posix default x509 path, or `None`
0092     """
0093     try:
0094         return '/tmp/x509up_u%s' % str(os.getuid())
0095     except AttributeError:
0096         logger.warn('No UID available? System not POSIX-compatible... trying to continue')
0097         pass
0098
0099     return None
0100
0101
0102 def cacert(args=None):
0103     """
0104     Tries to get :abbr:`CA (Certification Authority)` certificate or X509 one.
0105     Testifies it to be a regular file.
0106     Tries next locations:
0107
0108     1. :option:`--cacert` from arguments
0109     2. :envvar:`X509_USER_PROXY` from env
0110     3. Path ``/tmp/x509up_uXXX``, where ``XXX`` refers to ``UID``
0111
0112     :param args: arguments, parsed by `argparse`
0113     :returns: `str` -- certificate file path, or `None`
0114     """
0115
0116     return _tester(os.path.isfile,
0117                    args and args.cacert,
0118                    os.environ.get('X509_USER_PROXY'),
0119                    cacert_default_location())
0120
0121
0122 def https_setup(args=None, version=None):
0123     """
0124     Sets up the context for future HTTPS requests:
0125
0126     1. Selects the certificate paths
0127     2. Sets up :mailheader:`User-Agent`
0128     3. Tries to create `ssl.SSLContext` for future use (falls back to :command:`curl` if fails)
0129
0130     :param args: arguments, parsed by `argparse`
0131     :param str version: pilot version string (for :mailheader:`User-Agent`)
0132     """
0133
0134     version = version or get_pilot_version()
0135
0136     _ctx.user_agent = 'pilot/%s (Python %s; %s %s)' % (version,
0137                                                        sys.version.split()[0],
0138                                                        platform.system(),
0139                                                        platform.machine())
0140     logger.debug('User-Agent: %s' % _ctx.user_agent)
0141
0142     _ctx.capath = capath(args)
0143     _ctx.cacert = cacert(args)
0144
0145     if sys.version_info < (2, 7, 9):  # by anisyonk: actually SSL context should work, but prior to 2.7.9 there is no automatic hostname/certificate validation
0146         logger.warn('Python version <2.7.9 lacks SSL contexts -- falling back to curl')
0147         _ctx.ssl_context = None
0148     else:
0149         try:
0150             _ctx.ssl_context = ssl.create_default_context(capath=_ctx.capath,
0151                                                           cafile=_ctx.cacert)
0152         except Exception as e:
0153             logger.warn('SSL communication is impossible due to SSL error: %s -- falling back to curl' % str(e))
0154             _ctx.ssl_context = None
0155
0156     # anisyonk: clone `_ctx` to avoid logic break since ssl_context is reset inside the request() -- FIXME
0157     ctx.capath = _ctx.capath
0158     ctx.cacert = _ctx.cacert
0159     ctx.user_agent = _ctx.user_agent
0160
0161     try:
0162         ctx.ssl_context = ssl.create_default_context(capath=ctx.capath, cafile=ctx.cacert)
0163         ctx.ssl_context.load_cert_chain(ctx.cacert)
0164     except Exception as e:  # redandant try-catch protection, should work well for both python2 & python3 -- CLEAN ME later (anisyonk)
0165         logger.warn('Failed to initialize SSL context .. skipped, error: %s' % str(e))
0166
0167
0168 def request(url, data=None, plain=False, secure=True):
0169     """
0170     This function sends a request using HTTPS.
0171     Sends :mailheader:`User-Agent` and certificates previously being set up by `https_setup`.
0172     If `ssl.SSLContext` is available, uses `urllib2` as a request processor. Otherwise uses :command:`curl`.
0173
0174     If ``data`` is provided, encodes it as a URL form data and sends it to the server.
0175
0176     Treats the request as JSON unless a parameter ``plain`` is `True`.
0177     If JSON is expected, sends ``Accept: application/json`` header.
0178
0179     :param string url: the URL of the resource
0180     :param dict data: data to send
0181     :param boolean plain: if true, treats the response as a plain text.
0182     :param secure: Boolean (default: True, ie use certificates)
0183     Usage:
0184
0185     .. code-block:: python
0186         :emphasize-lines: 2
0187
0188         https_setup(args, PILOT_VERSION)  # sets up ssl and other stuff
0189         response = request('https://some.url', {'some':'data'})
0190
0191     Returns:
0192         - :keyword:`dict` -- if everything went OK
0193         - `str` -- if ``plain`` parameter is `True`
0194         - `None` -- if something went wrong
0195     """
0196
0197     _ctx.ssl_context = None  # certificates are not available on the grid, use curl
0198
0199     logger.debug('server update dictionary = \n%s' % str(data))
0200
0201     # get the filename and strdata for the curl config file
0202     filename, strdata = get_vars(url, data)
0203     # write the strdata to file
0204     writestatus = write_file(filename, strdata)
0205     # get the config option for the curl command
0206     dat = get_curl_config_option(writestatus, url, data, filename)
0207
0208     if _ctx.ssl_context is None and secure:
0209         req = get_curl_command(plain, dat)
0210
0211         try:
0212             status, output = execute_request(req)
0213         except Exception as e:
0214             logger.warning('exception: %s' % e)
0215             return None
0216         else:
0217             if status != 0:
0218                 logger.warn('request failed (%s): %s' % (status, output))
0219                 return None
0220
0221         # return output if plain otherwise return json.loads(output)
0222         if plain:
0223             return output
0224         else:
0225             try:
0226                 ret = json.loads(output)
0227             except Exception as e:
0228                 logger.warning('json.loads() failed to parse output=%s: %s' % (output, e))
0229                 return None
0230             else:
0231                 return ret
0232     else:
0233         req = execute_urllib(url, data, plain, secure)
0234         context = _ctx.ssl_context if secure else None
0235
0236         if is_python3():  # Python 3
0237             ec, output = get_urlopen_output(req, context)
0238             if ec:
0239                 return None
0240         else:  # Python 2
0241             ec, output = get_urlopen2_output(req, context)
0242             if ec:
0243                 return None
0244
0245         return output.read() if plain else json.load(output)
0246
0247
0248 def get_curl_command(plain, dat):
0249     """
0250     Get the curl command.
0251
0252     :param plain:
0253     :param dat: curl config option (string).
0254     :return: curl command (string).
0255     """
0256     req = 'curl -sS --compressed --connect-timeout %s --max-time %s '\
0257           '--capath %s --cert %s --cacert %s --key %s '\
0258           '-H %s %s %s' % (config.Pilot.http_connect_timeout, config.Pilot.http_maxtime,
0259                            pipes.quote(_ctx.capath or ''), pipes.quote(_ctx.cacert or ''),
0260                            pipes.quote(_ctx.cacert or ''), pipes.quote(_ctx.cacert or ''),
0261                            pipes.quote('User-Agent: %s' % _ctx.user_agent),
0262                            "-H " + pipes.quote('Accept: application/json') if not plain else '',
0263                            dat)
0264     logger.info('request: %s' % req)
0265     return req
0266
0267
0268 def get_vars(url, data):
0269     """
0270     Get the filename and strdata for the curl config file.
0271
0272     :param url: URL (string).
0273     :param data: data to be written to file (dictionary).
0274     :return: filename (string), strdata (string).
0275     """
0276
0277     strdata = ""
0278     for key in data:
0279         try:
0280             strdata += 'data="%s"\n' % urllib.parse.urlencode({key: data[key]})  # Python 3
0281         except Exception:
0282             strdata += 'data="%s"\n' % urllib.urlencode({key: data[key]})  # Python 2
0283     jobid = ''
0284     if 'jobId' in list(data.keys()):  # Python 2/3
0285         jobid = '_%s' % data['jobId']
0286
0287     # write data to temporary config file
0288     filename = '%s/curl_%s%s.config' % (os.getenv('PILOT_HOME'), os.path.basename(url), jobid)
0289
0290     return filename, strdata
0291
0292
0293 def get_curl_config_option(writestatus, url, data, filename):
0294     """
0295     Get the curl config option.
0296
0297     :param writestatus: status of write_file call (Boolean).
0298     :param url: URL (string).
0299     :param data: data structure (dictionary).
0300     :param filename: file name of config file (string).
0301     :return: config option (string).
0302     """
0303
0304     if not writestatus:
0305         logger.warning('failed to create curl config file (will attempt to urlencode data directly)')
0306         try:
0307             dat = pipes.quote(url + '?' + urllib.parse.urlencode(data) if data else '')  # Python 3
0308         except Exception:
0309             dat = pipes.quote(url + '?' + urllib.urlencode(data) if data else '')  # Python 2
0310     else:
0311         dat = '--config %s %s' % (filename, url)
0312
0313     return dat
0314
0315
0316 def execute_request(req):
0317     """
0318     Execute the curl request.
0319
0320     :param req: curl request command (string).
0321     :return: status (int), output (string).
0322     """
0323     try:
0324         status, output = subprocess.getstatusoutput(req)  # Python 3
0325     except Exception:
0326         status, output = commands.getstatusoutput(req)  # Python 2
0327     return status, output
0328
0329
0330 def execute_urllib(url, data, plain, secure):
0331     """
0332     Execute the request using urllib.
0333
0334     :param url: URL (string).
0335     :param data: data structure
0336     :return: urllib request structure.
0337     """
0338     try:
0339         req = urllib.request.Request(url, urllib.parse.urlencode(data))  # Python 3
0340     except Exception:
0341         req = urllib2.Request(url, urllib.urlencode(data))  # Python 2
0342
0343     if not plain:
0344         req.add_header('Accept', 'application/json')
0345     if secure:
0346         req.add_header('User-Agent', _ctx.user_agent)
0347
0348     return req
0349
0350
0351 def get_urlopen_output(req, context):
0352     """
0353     Get the output from the urlopen request.
0354
0355     :param req:
0356     :param context:
0357     :return: ec (int), output (string).
0358     """
0359
0360     ec = -1
0361     output = ""
0362     try:
0363         output = urllib.request.urlopen(req, context=context)
0364     except urllib.error.HTTPError as e:
0365         logger.warn('server error (%s): %s' % (e.code, e.read()))
0366     except urllib.error.URLError as e:
0367         logger.warn('connection error: %s' % e.reason)
0368     else:
0369         ec = 0
0370
0371     return ec, output
0372
0373
0374 def get_urlopen2_output(req, context):
0375     """
0376     Get the output from the urlopen2 request.
0377
0378     :param req:
0379     :param context:
0380     :return: ec (int), output (string).
0381     """
0382
0383     ec = -1
0384     output = ""
0385     try:
0386         output = urllib2.urlopen(req, context=context)
0387     except urllib2.HTTPError as e:
0388         logger.warn('server error (%s): %s' % (e.code, e.read()))
0389     except urllib2.URLError as e:
0390         logger.warn('connection error: %s' % e.reason)
0391     else:
0392         ec = 0
0393
0394     return ec, output