idds/tests/common.py

0001 #!/usr/bin/env python
0002 #
0003 # Licensed under the Apache License, Version 2.0 (the "License");
0004 # You may not use this file except in compliance with the License.
0005 # You may obtain a copy of the License at
0006 # http://www.apache.org/licenses/LICENSE-2.0OA
0007 #
0008 # Authors:
0009 # - Wen Guan, <wen.guan@cern.ch>, 2019
0010
0011 """
0012 common funcs for tests
0013 """
0014
0015 import random
0016 import time
0017 import datetime
0018 from uuid import uuid4 as uuid
0019
0020 from idds.common.constants import (RequestType, RequestStatus,
0021                                    TransformType, TransformStatus, CollectionType,
0022                                    CollectionRelationType, CollectionStatus,
0023                                    ContentType, ContentStatus,
0024                                    ProcessingStatus, GranularityType)
0025
0026
0027 def get_request_properties():
0028     req_properties = {
0029         'scope': 'test_scope',
0030         'name': 'test_name_%s' % str(uuid()),
0031         'requester': 'panda',
0032         'request_type': RequestType.EventStreaming,
0033         'transform_tag': 's3218',
0034         'workload_id': int(time.time()) + random.randint(0, 1000000),
0035         'status': RequestStatus.New,
0036         'priority': 0,
0037         'lifetime': 30,
0038         'request_metadata': {'workload_id1': int(time.time())}
0039     }
0040     return req_properties
0041
0042
0043 def get_transform_properties():
0044     trans_properties = {
0045         'transform_type': TransformType.EventStreaming,
0046         'transform_tag': 's3128',
0047         'priority': 0,
0048         'status': TransformStatus.New,
0049         'retries': 0,
0050         'expired_at': datetime.datetime.utcnow().replace(microsecond=0),
0051         'transform_metadata': {'input': {'coll_id': 123},
0052                                'output': {'coll_id': 456},
0053                                'log': {'coll_id': 789}}
0054     }
0055     return trans_properties
0056
0057
0058 def get_collection_properties():
0059     coll_properties = {
0060         'scope': 'test_scope',
0061         'name': 'test_name_%s' % str(uuid()),
0062         'coll_type': CollectionType.Dataset,
0063         'transform_id': None,
0064         'relation_type': CollectionRelationType.Input,
0065         'bytes': 0,
0066         'status': CollectionStatus.New,
0067         'total_files': 0,
0068         'retries': 0,
0069         'expired_at': datetime.datetime.utcnow().replace(microsecond=0),
0070         'coll_metadata': {'ddm_status': 'closed'}
0071     }
0072     return coll_properties
0073
0074
0075 def get_content_properties():
0076     content_properties = {
0077         'coll_id': None,
0078         'scope': 'test_scope',
0079         'name': 'test_file_name_%s' % str(uuid()),
0080         'min_id': 0,
0081         'max_id': 100,
0082         'content_type': ContentType.File,
0083         'status': ContentStatus.New,
0084         'bytes': 1,
0085         'md5': None,
0086         'adler32': None,
0087         'processing_id': None,
0088         'storage_id': None,
0089         'retries': 0,
0090         'path': None,
0091         'expired_at': datetime.datetime.utcnow().replace(microsecond=0),
0092         'content_metadata': {'id': 123}
0093     }
0094     return content_properties
0095
0096
0097 def get_processing_properties():
0098     proc_properties = {
0099         'transform_id': None,
0100         'status': ProcessingStatus.New,
0101         'submitter': 'panda',
0102         'granularity': 10,
0103         'granularity_type': GranularityType.File,
0104         'expired_at': datetime.datetime.utcnow().replace(microsecond=0),
0105         'processing_metadata': {'task_id': 40191323}
0106     }
0107     return proc_properties
0108
0109
0110 def get_example_tape_stagein_request():
0111     req_properties = {
0112         'scope': 'test_scope',
0113         'name': 'test_name_%s' % str(uuid()),
0114         'requester': 'panda',
0115         'request_type': RequestType.StageIn,
0116         'transform_tag': RequestType.StageIn.value,
0117         'status': RequestStatus.New,
0118         'priority': 0,
0119         'lifetime': 30,
0120         'request_metadata': {'workload_id': int(time.time()), 'src_rse': 'BNL-OSG2_DATATAPE', 'dest_rse': 'BNL-OSG2_DATADISK'}
0121     }
0122     return req_properties
0123
0124
0125 def get_example_real_tape_stagein_request():
0126     req_properties = {
0127         'scope': 'data17_13TeV',
0128         'name': 'data17_13TeV.00341649.express_express.merge.RAW',
0129         'requester': 'panda',
0130         'request_type': RequestType.StageIn,
0131         'transform_tag': RequestType.StageIn.value,
0132         'status': RequestStatus.New,
0133         'priority': 0,
0134         'lifetime': 30,
0135         'request_metadata': {'workload_id': int(time.time()), 'src_rse': 'BNL-OSG2_DATATAPE', 'dest_rse': 'BNL-OSG2_DATADISK'}
0136     }
0137     return req_properties
0138
0139
0140 def get_example_prodsys2_tape_stagein_request_old():
0141     req_properties = {
0142         'scope': 'data16_13TeV',
0143         'name': 'data16_13TeV.00298773.physics_Main.daq.RAW',
0144         'requester': 'panda',
0145         'request_type': RequestType.StageIn,
0146         'transform_tag': 'prodsys2',
0147         'status': RequestStatus.New,
0148         'priority': 0,
0149         'lifetime': 30,
0150         'request_metadata': {'workload_id': '20776834', 'src_rse': 'INFN-T1_DATATAPE', 'rule_id': 'ececf892223142b1b7c25b7593f50fe1'}
0151     }
0152     return req_properties
0153
0154
0155 def get_example_prodsys2_tape_stagein_request():
0156     req_properties = {
0157         'scope': 'data16_13TeV',
0158         'name': 'data16_13TeV.00301915.physics_Main.daq.RAW',
0159         'requester': 'panda',
0160         'request_type': RequestType.StageIn,
0161         'transform_tag': 'prodsys2',
0162         'status': RequestStatus.New,
0163         'priority': 0,
0164         'lifetime': 30,
0165         'request_metadata': {'workload_id': '20776936', 'src_rse': 'NDGF-T1_DATATAPE', 'rule_id': '1d522d631327478db5773b554f65b0ff'}
0166     }
0167     return req_properties
0168
0169
0170 def get_example_active_learning_request():
0171     req_properties = {
0172         'scope': 'data15_13TeV',
0173         'name': 'data15_13TeV.00270949.physics_Main.merge.AOD.r7600_p2521_tid07734829_00',
0174         'requester': 'panda',
0175         'request_type': RequestType.ActiveLearning,
0176         'transform_tag': 'prodsys2',
0177         'status': RequestStatus.New,
0178         'priority': 0,
0179         'lifetime': 30,
0180         'request_metadata': {'workload_id': '20525134', 'sandbox': 'https://', 'executable': 'hostname', 'arguments': '-s'}
0181     }
0182     return req_properties
0183
0184
0185 def get_example_ess_request():
0186     req_properties = {
0187         'scope': 'test_scope',
0188         'name': 'test_name_%s' % str(uuid()),
0189         'requester': 'panda',
0190         'request_type': RequestType.EventStreaming,
0191         'transform_tag': 's3128',
0192         'status': RequestStatus.New,
0193         'priority': 0,
0194         'lifetime': 30,
0195         'request_metadata': {'workload_id': int(time.time()), 'events_per_range': 10}
0196     }
0197     return req_properties
0198
0199
0200 # https://bigpanda.cern.ch/tasknew/19163542/
0201 def get_real_example_ess_request():
0202     req_properties = {
0203         'scope': 'mc16_13TeV',
0204         'name': 'mc16_13TeV.451129.MGPy8EG_A14N23LO_VBF_RS_G_ZZ_llqq_kt1_m1500.merge.EVNT.e7758_e5984_tid19153490_00',
0205         'requester': 'panda',
0206         'request_type': RequestType.EventStreaming,
0207         'transform_tag': 's3126',
0208         'status': RequestStatus.New,
0209         'priority': 0,
0210         'lifetime': 30,
0211         'request_metadata': {'workload_id': 19163542, 'events_per_range': 10}
0212     }
0213     return req_properties
0214
0215
0216 # https://bigpanda.cern.ch/task/19660169/
0217 def get_example_derivation_request():
0218     req_properties = {
0219         'scope': 'mc16_13TeV',
0220         'name': 'mc16_13TeV.410287.PhPy8EG_A14_ttbar_hdamp258p75_allhad_mtt_1700_2000.merge.AOD.e6686_e5984_s3126_r10724_r10726_tid15801996_00',
0221         'requester': 'panda',
0222         'request_type': RequestType.Derivation,
0223         'transform_tag': 'p3978',
0224         'status': RequestStatus.New,
0225         'priority': 0,
0226         'lifetime': 30,
0227         'request_metadata': {'workload_id': 19660169, 'task_parameter': None}
0228     }
0229     return req_properties
0230
0231
0232 def merge_dicts(list_dicts):
0233     ret = {}
0234     for a_dict in list_dicts:
0235         ret.update(a_dict)
0236     return ret
0237
0238
0239 def is_same_req_trans_colls(req_trans_colls1, req_trans_colls2, allow_request_id_None=False):
0240     req_ids1 = list(req_trans_colls1.keys())
0241     req_ids2 = list(req_trans_colls2.keys())
0242     if not allow_request_id_None and not (req_ids1 == req_ids2):
0243         return False
0244
0245     if not allow_request_id_None:
0246         for req_id in req_ids1:
0247             tran_ids1 = list(req_trans_colls1[req_id].keys())
0248             tran_ids2 = list(req_trans_colls2[req_id].keys())
0249             if not tran_ids1 == tran_ids2:
0250                 return False
0251
0252             for tran_id in tran_ids1:
0253                 colls1 = req_trans_colls1[req_id][tran_id]
0254                 colls2 = req_trans_colls2[req_id][tran_id]
0255                 coll_ids1 = [coll['coll_id'] for coll in colls1]
0256                 coll_ids2 = [coll['coll_id'] for coll in colls2]
0257                 coll_ids1.sort()
0258                 coll_ids2.sort()
0259                 if not coll_ids1 == coll_ids2:
0260                     return False
0261     else:
0262         trans1 = merge_dicts(req_trans_colls1.values())
0263         trans2 = merge_dicts(req_trans_colls2.values())
0264         tran_ids1 = list(trans1.keys())
0265         tran_ids2 = list(trans2.keys())
0266         if not tran_ids1 == tran_ids2:
0267             return False
0268
0269         for tran_id in tran_ids1:
0270             colls1 = trans1[tran_id]
0271             colls2 = trans2[tran_id]
0272             coll_ids1 = [coll['coll_id'] for coll in colls1]
0273             coll_ids2 = [coll['coll_id'] for coll in colls2]
0274             coll_ids1.sort()
0275             coll_ids2.sort()
0276             if not coll_ids1 == coll_ids2:
0277                 return False
0278     return True
0279
0280
0281 def is_same_req_trans_coll_contents(req_trans_colls1, req_trans_colls2, allow_request_id_None=True):
0282     req_ids1 = list(req_trans_colls1.keys())
0283     req_ids2 = list(req_trans_colls2.keys())
0284     if not allow_request_id_None and not (req_ids1 == req_ids2):
0285         return False
0286
0287     if not allow_request_id_None:
0288         for req_id in req_ids1:
0289             tran_ids1 = list(req_trans_colls1[req_id].keys())
0290             tran_ids2 = list(req_trans_colls2[req_id].keys())
0291             if not tran_ids1 == tran_ids2:
0292                 return False
0293
0294             for tran_id in tran_ids1:
0295                 coll_contents1 = req_trans_colls1[req_id][tran_id]
0296                 coll_contents2 = req_trans_colls2[req_id][tran_id]
0297                 coll_scope_names1 = [scope_name for scope_name in coll_contents1]
0298                 coll_scope_names2 = [scope_name for scope_name in coll_contents2]
0299                 coll_scope_names1.sort()
0300                 coll_scope_names2.sort()
0301                 if not coll_scope_names1 == coll_scope_names1:
0302                     return False
0303
0304                 for scope_name in coll_scope_names1:
0305                     contents1 = req_trans_colls1[req_id][tran_id][scope_name]['contents']
0306                     contents2 = req_trans_colls2[req_id][tran_id][scope_name]['contents']
0307                     content_ids1 = [content['content_id'] for content in contents1]
0308                     content_ids2 = [content['content_id'] for content in contents2]
0309                     if not content_ids1 == content_ids2:
0310                         return False
0311     else:
0312         if True:
0313             trans1 = merge_dicts(req_trans_colls1.values())
0314             trans2 = merge_dicts(req_trans_colls2.values())
0315             tran_ids1 = list(trans1.keys())
0316             tran_ids2 = list(trans2.keys())
0317             if not tran_ids1 == tran_ids2:
0318                 return False
0319
0320             for tran_id in tran_ids1:
0321                 coll_contents1 = trans1[tran_id]
0322                 coll_contents2 = trans2[tran_id]
0323                 coll_scope_names1 = [scope_name for scope_name in coll_contents1]
0324                 coll_scope_names2 = [scope_name for scope_name in coll_contents2]
0325                 coll_scope_names1.sort()
0326                 coll_scope_names2.sort()
0327                 if not coll_scope_names1 == coll_scope_names1:
0328                     return False
0329
0330                 for scope_name in coll_scope_names1:
0331                     contents1 = trans1[tran_id][scope_name]['contents']
0332                     contents2 = trans2[tran_id][scope_name]['contents']
0333                     content_ids1 = [content['content_id'] for content in contents1]
0334                     content_ids2 = [content['content_id'] for content in contents2]
0335                     if not content_ids1 == content_ids2:
0336                         return False
0337     return True