idds/tests/hyperparameteropt_docker_local_test.py

0001 #!/usr/bin/env python
0002 #
0003 # Licensed under the Apache License, Version 2.0 (the "License");
0004 # You may not use this file except in compliance with the License.
0005 # You may obtain a copy of the License at
0006 # http://www.apache.org/licenses/LICENSE-2.0OA
0007 #
0008 # Authors:
0009 # - Wen Guan, <wen.guan@cern.ch>, 2020
0010
0011
0012 """
0013 Test hyper parameter optimization docker container in local env.
0014 """
0015
0016 import json
0017 import os
0018 import random
0019 import re
0020 import subprocess
0021 import tempfile
0022 # from uuid import uuid4 as uuid
0023
0024 # from idds.client.client import Client
0025 # from idds.common.utils import replace_parameters_with_values, run_command
0026 from idds.common.constants import RequestType, RequestStatus
0027 # from idds.common.utils import get_rest_host
0028
0029
0030 def get_test_codes():
0031     dir_name = os.path.dirname(os.path.abspath(__file__))
0032     test_codes = os.path.join(dir_name, 'activelearning_test_codes/activelearning_test_codes.tgz')
0033     return test_codes
0034
0035
0036 def run_command(cmd):
0037     """
0038     Runs a command in an out-of-procees shell.
0039     """
0040     process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid)
0041     stdout, stderr = process.communicate()
0042     if stdout is not None and type(stdout) in [bytes]:
0043         stdout = stdout.decode()
0044     if stderr is not None and type(stderr) in [bytes]:
0045         stderr = stderr.decode()
0046     status = process.returncode
0047     return status, stdout, stderr
0048
0049
0050 def replace_parameters_with_values(text, values):
0051     """
0052     Replace all strings starting with '%'. For example, for this string below, it should replace ['%NUM_POINTS', '%IN', '%OUT']
0053     'run --rm -it -v "$(pwd)":/payload gitlab-registry.cern.ch/zhangruihpc/endpointcontainer:latest /bin/bash -c "echo "--num_points %NUM_POINTS"; /bin/cat /payload/%IN>/payload/%OUT"'
0054
0055     :param text
0056     :param values: parameter values, for example {'NUM_POINTS': 5, 'IN': 'input.json', 'OUT': 'output.json'}
0057     """
0058     for key in values:
0059         key1 = '%' + key
0060         text = re.sub(key1, str(values[key]), text)
0061     return text
0062
0063
0064 def create_temp_dir():
0065     dirpath = tempfile.mkdtemp()
0066     return dirpath
0067
0068
0069 def create_idds_input_json(job_dir, req_meta):
0070     input_json = 'idds_input_%s.json' % random.randint(1, 1000)
0071     if 'initial_points' in req_meta:
0072         points = req_meta['initial_points']
0073     else:
0074         points = []
0075
0076     opt_space = None
0077     opt_points = {'points': points, 'opt_space': opt_space}
0078     if 'opt_space' in req_meta:
0079         opt_space = req_meta['opt_space']
0080         opt_points['opt_space'] = opt_space
0081     with open(os.path.join(job_dir, input_json), 'w') as f:
0082         json.dump(opt_points, f)
0083     return input_json
0084
0085
0086 def get_output_json(req_meta):
0087     output_json = None
0088     if 'output_json' in req_meta:
0089         output_json = req_meta['output_json']
0090     return output_json
0091
0092
0093 def get_exec(job_dir, req_meta):
0094     if 'max_points' in req_meta:
0095         max_points = req_meta['max_points']
0096     else:
0097         max_points = 20
0098     if 'num_points_per_generation' in req_meta:
0099         num_points = req_meta['num_points_per_generation']
0100     else:
0101         num_points = 10
0102     num_init_points = 0
0103     if 'initial_points' in req_meta:
0104         num_init_points = len(req_meta['initial_points'])
0105     num_points = min(max_points, num_points) - num_init_points
0106
0107     idds_input_json = create_idds_input_json(job_dir, req_meta)
0108     output_json = get_output_json(req_meta)
0109
0110     param_values = {'MAX_POINTS': max_points,
0111                     'NUM_POINTS': num_points,
0112                     'IN': idds_input_json,
0113                     'OUT': output_json}
0114
0115     executable = req_meta['executable'].strip()
0116     arguments = req_meta['arguments']
0117     executable = replace_parameters_with_values(executable, param_values)
0118     arguments = replace_parameters_with_values(arguments, param_values)
0119
0120     if executable == 'docker':
0121         executable = 'docker run --rm -v $(pwd):%s %s' % (req_meta['workdir'], req_meta['sandbox'])
0122     exect = executable + ' ' + arguments
0123     return exect
0124
0125
0126 def get_req_properties():
0127     req_properties = {
0128         # 'scope': 'data15_13TeV',
0129         # 'name': 'data15_13TeV.00270949.pseudo.%s' % str(uuid()),
0130         'requester': 'panda',
0131         'request_type': RequestType.HyperParameterOpt,
0132         'transform_tag': 'prodsys2',
0133         'status': RequestStatus.New,
0134         'priority': 0,
0135         'lifetime': 30,
0136         # 'request_metadata': {'workload_id': '20525134', 'sandbox': None, 'executable': 'docker run --rm -it -v "$(pwd)":/payload gitlab-registry.cern.ch/zhangruihpc/endpointcontainer:latest /bin/bash -c "/bin/cat /payload/input_json.txt>/payload/output_json.txt"', 'arguments': '-s --input %IN', 'output_json': 'output.json'}  # noqa: E501
0137         # 'request_metadata': {'workload_id': '20525134', 'is_pseudo_input': True, 'sandbox': None, 'executable': 'docker', 'arguments': 'run --rm -it -v "$(pwd)":/payload gitlab-registry.cern.ch/zhangruihpc/endpointcontainer:latest /bin/cat /payload/%IN>%OUT', 'initial_points': [({'A': 1, 'B': 2}, 0.3), ({'A': 1, 'B': 3}, None)], 'output_json': 'output.json'}  # noqa: E501
0138         # 'request_metadata': {'workload_id': '20525134', 'is_pseudo_input': True, 'sandbox': None, 'method': 'nevergrad', 'opt_space': {'A': [1, 2, 3], 'B': (1, 10)}, 'initial_points': [({'A': 1, 'B': 2}, 0.3), ({'A': 1, 'B': 3}, None)], 'max_points': 10}  # noqa: E501
0139         # 'request_metadata': {'workload_id': '20525134', 'sandbox': None, 'executable': 'docker', 'arguments': 'run -v $(pwd):/data wguanicedew/idds_hpo_nevergrad python /opt/hyperparameteropt_nevergrad.py --max_points=%MAX_POINTS --num_points=%NUM_POINTS --input=/data/%IN --output=/data/%OUT', 'output_json': 'output.json', 'opt_space': {"A": {"type": "Choice", "params": {"choices": [1, 4]}}, "B": {"type": "Scalar", "bounds": [0, 5]}}, 'initial_points': [({'A': 1, 'B': 2}, 0.3), ({'A': 1, 'B': 3}, None)], 'max_points': 20, 'num_points_per_generation': 10}  # noqa: E501
0140         'request_metadata': {'workload_id': '20525134', 'sandbox': 'wguanicedew/idds_hpo_nevergrad', 'workdir': '/data', 'executable': 'docker', 'arguments': 'python /opt/hyperparameteropt_nevergrad.py --max_points=%MAX_POINTS --num_points=%NUM_POINTS --input=/data/%IN --output=/data/%OUT', 'output_json': 'output.json', 'opt_space': {"A": {"type": "Choice", "params": {"choices": [1, 4]}}, "B": {"type": "Scalar", "bounds": [0, 5]}}, 'initial_points': [({'A': 1, 'B': 2}, 0.3), ({'A': 1, 'B': 3}, None)], 'max_points': 20, 'num_points_per_generation': 10}  # noqa: E501
0141     }
0142     return req_properties
0143
0144
0145 def run_test():
0146     job_dir = create_temp_dir()
0147     print('job dir: %s' % job_dir)
0148     os.chdir(job_dir)
0149     print("$pwd: %s" % os.getcwd())
0150     req = get_req_properties()
0151     req_meta = req['request_metadata']
0152     exe = get_exec(job_dir, req_meta)
0153     print('executable: %s' % exe)
0154     status, output, error = run_command(exe)
0155     print('status: %s, output: %s, error: %s' % (status, output, error))
0156     output_json = get_output_json(req_meta)
0157     if not output_json:
0158         print('output_json is not defined')
0159     else:
0160         output_json = os.path.join(job_dir, output_json)
0161         if not os.path.exists(output_json):
0162             print("Failed: output_json is not created")
0163         else:
0164             try:
0165                 with open(output_json, 'r') as f:
0166                     data = f.read()
0167                 outputs = json.loads(data)
0168                 print("outputs: %s" % str(outputs))
0169                 if type(outputs) not in [list, tuple]:
0170                     print("Failed: outputs is not a list")
0171             except Exception as ex:
0172                 print("Failed to parse output_json: %s" % ex)
0173
0174
0175 if __name__ == '__main__':
0176     run_test()