idds/tests/datacarousel_test.py

0001 #!/usr/bin/env python
0002 #
0003 # Licensed under the Apache License, Version 2.0 (the "License");
0004 # You may not use this file except in compliance with the License.
0005 # You may obtain a copy of the License at
0006 # http://www.apache.org/licenses/LICENSE-2.0OA
0007 #
0008 # Authors:
0009 # - Wen Guan, <wen.guan@cern.ch>, 2019 - 2020
0010
0011
0012 """
0013 Test client.
0014 """
0015
0016 import traceback
0017
0018 from rucio.client.client import Client as Rucio_Client
0019 from rucio.common.exception import CannotAuthenticate
0020
0021 from idds.client.client import Client
0022 from idds.common.constants import RequestType, RequestStatus
0023 from idds.common.utils import get_rest_host
0024 # from idds.tests.common import get_example_real_tape_stagein_request
0025 # from idds.tests.common import get_example_prodsys2_tape_stagein_request
0026
0027
0028 def get_rucio_client():
0029     try:
0030         client = Rucio_Client()
0031     except CannotAuthenticate as error:
0032         print(traceback.format_exc())
0033         raise Exception('%s: %s' % (str(error), traceback.format_exc()))
0034     return client
0035
0036
0037 def get_rule(scope, name, rucio_client, src_rse, dest_rse, account='ddmadmin'):
0038     rules = rucio_client.list_did_rules(scope=scope, name=name)
0039     for rule in rules:
0040         if rule['source_replica_expression'] == src_rse and rule['rse_expression']:
0041             print(rule['id'])
0042             return rule['id']
0043         # print(rule)
0044     return None
0045
0046
0047 def create_rule(scope, name, rucio_client, src_rse, dest_rse, account='ddmadmin'):
0048     did = {'scope': scope, 'name': name}
0049     rule_id = rucio_client.add_replication_rule(dids=[did],
0050                                                 copies=1,
0051                                                 rse_expression=dest_rse,
0052                                                 source_replica_expression=src_rse,
0053                                                 lifetime=24 * 7 * 3600,
0054                                                 locked=False,
0055                                                 account=account,
0056                                                 grouping='DATASET',
0057                                                 ask_approval=False)
0058     return rule_id
0059
0060
0061 # max_waiting_time is used for idds to create new rules
0062 def get_req_properties_old():
0063     req_properties = {
0064         'scope': 'data16_13TeV',
0065         'name': 'data16_13TeV.00298862.physics_Main.daq.RAW',
0066         'requester': 'panda',
0067         'request_type': RequestType.StageIn,
0068         'transform_tag': 'prodsys2',
0069         'status': RequestStatus.New,
0070         'priority': 0,
0071         'lifetime': 30,
0072         'request_metadata': {'workload_id': '20776840', 'max_waiting_time': 3600, 'src_rse': 'NDGF-T1_DATATAPE', 'dest_rse': 'NDGF-T1_DATADISK', 'rule_id': '236e4bf87e11490291e3259b14724e30'}
0073     }
0074     return req_properties
0075
0076
0077 def get_req_properties():
0078     req_properties = {
0079         'scope': 'data15_13TeV',
0080         'name': 'data15_13TeV.00284484.physics_Main.merge.DRAW_EGZ.f644_m1453',
0081         'requester': 'panda',
0082         'request_type': RequestType.StageIn,
0083         'transform_tag': 'prodsys2',
0084         'status': RequestStatus.New,
0085         'priority': 0,
0086         'lifetime': 30,
0087         'request_metadata': {'workload_id': '44158478', 'max_waiting_time': 3600, 'src_rse': 'BNL-OSG2_DATADISK', 'dest_rse': 'BNL-OSG2_DATADISK', 'rule_id': 'fd30d4b9e9744242a06f725bf40fc890'}
0088     }
0089     return req_properties
0090
0091
0092 def pre_check(req):
0093     rucio_client = get_rucio_client()
0094     rule_id = get_rule(req['scope'], req['name'], rucio_client, req['request_metadata']['src_rse'], req['request_metadata']['dest_rse'])
0095     if not rule_id:
0096         rule_id = create_rule(req['scope'], req['name'], rucio_client, req['request_metadata']['src_rse'], req['request_metadata']['dest_rse'])
0097     if rule_id:
0098         print("new rule id: %s" % rule_id)
0099         req['request_metadata']['rule_id'] = rule_id
0100     return req
0101
0102
0103 host = get_rest_host()
0104 props = get_req_properties()
0105 # props = get_example_real_tape_stagein_request()
0106 # props = get_example_prodsys2_tape_stagein_request()
0107 # props = get_example_active_learning_request()
0108
0109 # props = pre_check(props)
0110 print(props)
0111
0112 client = Client(host=host, client_proxy='/tmp/x509up_u23959')
0113
0114 request_id = client.add_request(**props)
0115 print(request_id)