idds/tests/test_migrate_requests.py

0001 #!/usr/bin/env python
0002 #
0003 # Licensed under the Apache License, Version 2.0 (the "License");
0004 # You may not use this file except in compliance with the License.
0005 # You may obtain a copy of the License at
0006 # http://www.apache.org/licenses/LICENSE-2.0OA
0007 #
0008 # Authors:
0009 # - Wen Guan, <wen.guan@cern.ch>, 2021
0010
0011
0012 """
0013 Test client.
0014 """
0015
0016
0017 from idds.client.clientmanager import ClientManager
0018 from idds.common.constants import RequestStatus
0019 from idds.common.utils import json_dumps  # noqa F401
0020 from idds.rest.v1.utils import convert_old_req_2_workflow_req  # noqa F401
0021 from idds.common.utils import setup_logging
0022
0023
0024 setup_logging("idds.log")
0025
0026
0027 def migrate():
0028     # 72533, 72569, 72733, 72769, 72783, 72939, 73351, 73983, 74545,
0029     # 74567, 74569, 74573
0030     # dev
0031     dev_host = 'https://aipanda160.cern.ch:443/idds'    # noqa F841
0032     # doma
0033     doma_host = 'https://aipanda015.cern.ch:443/idds'   # noqa F841
0034     # atlas
0035     atlas_host = 'https://aipanda181.cern.ch:443/idds'  # noqa F841
0036     # doma google
0037     doma_google_host = 'https://34.133.138.229:443/idds'  # noqa F841
0038
0039     slac_k8s_dev_host = 'https://rubin-panda-idds-dev.slac.stanford.edu:8443/idds'  # noqa F841
0040
0041     slac_k8s_prod_host = 'https://usdf-panda-idds.slac.stanford.edu:8443/idds'  # noqa F841
0042
0043     cern_k8s_dev_host = 'https://panda-idds-dev.cern.ch/idds'  # noqa F841
0044
0045     # cm1 = ClientManager(host=atlas_host)
0046     # cm1 = ClientManager(host=doma_host)
0047     # cm1 = ClientManager(host=atlas_host)
0048     cm1 = ClientManager(host=slac_k8s_dev_host)
0049     # cm1 = ClientManager(host=slac_k8s_prod_host)
0050     # reqs = cm1.get_requests(request_id=290)
0051     # old_request_id = 298163
0052     # old_request_id = 350723
0053     old_request_id = 359383
0054     # old_request_id = 349
0055     old_request_id = 2400
0056     old_request_id = 371204
0057     old_request_id = 372930
0058     old_request_id = 2603
0059     old_request_id = 2802
0060     old_request_id = 2816       # big tasks
0061     # old_request_id = 3178     # 125 tasks
0062     old_request_id = 3578
0063     old_request_id = 3612
0064     old_request_id = 3628
0065
0066     old_request_ids = [3628]
0067
0068     # old_request_ids = [21]
0069
0070     old_request_ids = [551889]
0071
0072     old_request_ids = [i for i in range(551911, 556618)]
0073     old_request_ids = [i for i in range(556618, 556712)]
0074     old_request_ids = [i for i in range(556712, 556940)]
0075     old_request_ids = [i for i in range(556940, 557110)]
0076     # old_request_id = 1
0077     # for old_request_id in [152]:
0078     # for old_request_id in [60]:    # noqa E115
0079     # for old_request_id in [200]:    # noqa E115
0080     old_request_ids = [8526]
0081     old_request_ids = [162704]
0082     for old_request_id in old_request_ids:    # noqa E115  # doma 183
0083         reqs = cm1.get_requests(request_id=old_request_id, with_metadata=True)
0084
0085         # cm2 = ClientManager(host=dev_host)
0086         # cm2 = ClientManager(host=doma_host)
0087         # cm2 = ClientManager(host=atlas_host)
0088         cm2 = ClientManager(host=slac_k8s_dev_host)
0089         # cm2 = ClientManager(host=slac_k8s_prod_host)
0090         # cm2 = ClientManager(host=cern_k8s_dev_host)
0091         # print(reqs)
0092
0093         print("num requests: %s" % len(reqs))
0094         for req in reqs[:1]:
0095             # print(req)
0096             # workflow = req['request_metadata']['workflow']
0097             # print(json_dumps(workflow, sort_keys=True, indent=4))
0098
0099             # req = convert_old_req_2_workflow_req(req)
0100             print(req['status'])
0101             if req['status'] in [RequestStatus.Finished]:
0102                 print('request stutus: %s, skip' % str(req['status']))
0103                 continue
0104
0105             workflow = req['request_metadata']['workflow']
0106             workflow.clean_works()
0107
0108             # for old idds version
0109             t_works = workflow.template.works
0110             if not t_works and hasattr(workflow, 'works_template'):
0111                 workflow.template.works = workflow.works_template
0112
0113             # print(json_dumps(workflow))
0114             # print(json_dumps(workflow, sort_keys=True, indent=4))
0115             req_id = cm2.submit(workflow)
0116             print("old request %s -> new request %s" % (old_request_id, req_id))
0117
0118
0119 if __name__ == '__main__':
0120     migrate()