idds/tests/test_catalog.py

0001 #!/usr/bin/env python
0002 #
0003 # Licensed under the Apache License, Version 2.0 (the "License");
0004 # You may not use this file except in compliance with the License.
0005 # You may obtain a copy of the License at
0006 # http://www.apache.org/licenses/LICENSE-2.0OA
0007 #
0008 # Authors:
0009 # - Wen Guan, <wen.guan@cern.ch>, 2019
0010
0011
0012 """
0013 Test Request.
0014 """
0015
0016 import copy
0017
0018 import unittest2 as unittest
0019 from nose.tools import assert_equal, assert_in
0020
0021 from idds.client.client import Client
0022 from idds.common.constants import (CollectionRelationType, ContentType, ContentStatus)
0023 from idds.common.utils import (check_database, has_config, setup_logging,
0024                                check_rest_host, get_rest_host, check_user_proxy)
0025 from idds.core.catalog import (get_collections, get_contents,
0026                                register_output_contents, get_match_contents)
0027 from idds.orm.requests import add_request
0028 from idds.orm.transforms import add_transform
0029 from idds.orm.collections import add_collection
0030 from idds.orm.contents import add_content, get_content
0031 from idds.tests.common import (get_request_properties, get_transform_properties,
0032                                get_collection_properties, get_content_properties,
0033                                is_same_req_trans_colls, is_same_req_trans_coll_contents)
0034
0035 setup_logging(__name__)
0036
0037
0038 class TestCatalog(unittest.TestCase):
0039
0040     @unittest.skipIf(not has_config(), "No config file")
0041     @unittest.skipIf(not check_database(), "Database is not defined")
0042     def test_catalog_core(self):
0043         """ Catalog (Core): Test catalog core functions """
0044         req_properties = get_request_properties()
0045         origin_request_id = add_request(**req_properties)
0046
0047         trans_properties = get_transform_properties()
0048         trans_properties['request_id'] = origin_request_id
0049         origin_trans_id = add_transform(**trans_properties)
0050
0051         coll_properties = get_collection_properties()
0052         coll_properties['transform_id'] = origin_trans_id
0053         coll_input_properties = copy.deepcopy(coll_properties)
0054         coll_input_properties['name'] = coll_input_properties['name'] + '_input'
0055         coll_output_properties = copy.deepcopy(coll_properties)
0056         coll_output_properties['relation_type'] = CollectionRelationType.Output
0057         coll_output_properties['name'] = coll_output_properties['name'] + '_output'
0058         coll_log_properties = copy.deepcopy(coll_properties)
0059         coll_log_properties['relation_type'] = CollectionRelationType.Log
0060         coll_log_properties['name'] = coll_log_properties['name'] + '_log'
0061
0062         origin_coll_input_id = add_collection(**coll_input_properties)
0063         origin_coll_output_id = add_collection(**coll_output_properties)
0064         origin_coll_log_id = add_collection(**coll_log_properties)
0065         origin_coll_id_list = [origin_coll_input_id, origin_coll_output_id, origin_coll_log_id]
0066
0067         req_trans_colls = get_collections(request_id=origin_request_id, workload_id=None)
0068         assert_equal(len(req_trans_colls.keys()), 1)
0069         req_id = list(req_trans_colls.keys())[0]
0070         assert_equal(origin_request_id, req_id)
0071         assert_equal(len(req_trans_colls[req_id].keys()), 1)
0072         trans_id = list(req_trans_colls[req_id].keys())[0]
0073         assert_equal(trans_id, origin_trans_id)
0074         colls = req_trans_colls[req_id][trans_id]
0075         assert_equal(len(colls), 3)
0076         for coll in colls:
0077             assert_in(coll['coll_id'], origin_coll_id_list)
0078             for key in coll_input_properties:
0079                 if key == 'relation_type':
0080                     if coll['coll_id'] == origin_coll_input_id:
0081                         assert_equal(coll[key], CollectionRelationType.Input)
0082                     elif coll['coll_id'] == origin_coll_output_id:
0083                         assert_equal(coll[key], CollectionRelationType.Output)
0084                     else:
0085                         assert_equal(coll['coll_id'], origin_coll_log_id)
0086                         assert_equal(coll[key], CollectionRelationType.Log)
0087                 if key == 'name':
0088                     if coll['coll_id'] == origin_coll_input_id:
0089                         assert_equal(coll[key], coll_input_properties[key])
0090                     elif coll['coll_id'] == origin_coll_output_id:
0091                         assert_equal(coll[key], coll_output_properties[key])
0092                     else:
0093                         assert_equal(coll[key], coll_log_properties[key])
0094
0095         req_trans_colls1 = get_collections(request_id=None, workload_id=req_properties['workload_id'])
0096         assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1, allow_request_id_None=True), True)
0097
0098         req_trans_colls1 = get_collections(request_id=origin_request_id, workload_id=req_properties['workload_id'])
0099         assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
0100
0101         req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_properties['name'] + "*", request_id=None, workload_id=None)
0102         assert_equal(len(req_trans_colls.keys()), 1)
0103         req_id = list(req_trans_colls.keys())[0]
0104         assert_equal(None, req_id)
0105         assert_equal(len(req_trans_colls[req_id].keys()), 1)
0106         trans_id = list(req_trans_colls[req_id].keys())[0]
0107         assert_equal(trans_id, origin_trans_id)
0108         colls = req_trans_colls[req_id][trans_id]
0109         assert_equal(len(colls), 3)
0110         for coll in colls:
0111             assert_in(coll['coll_id'], origin_coll_id_list)
0112             for key in coll_input_properties:
0113                 if key == 'relation_type':
0114                     if coll['coll_id'] == origin_coll_input_id:
0115                         assert_equal(coll[key], CollectionRelationType.Input)
0116                     elif coll['coll_id'] == origin_coll_output_id:
0117                         assert_equal(coll[key], CollectionRelationType.Output)
0118                     else:
0119                         assert_equal(coll['coll_id'], origin_coll_log_id)
0120                         assert_equal(coll[key], CollectionRelationType.Log)
0121                 if key == 'name':
0122                     if coll['coll_id'] == origin_coll_input_id:
0123                         assert_equal(coll[key], coll_input_properties[key])
0124                     elif coll['coll_id'] == origin_coll_output_id:
0125                         assert_equal(coll[key], coll_output_properties[key])
0126                     else:
0127                         assert_equal(coll[key], coll_log_properties[key])
0128
0129         req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_input_properties['name'], request_id=origin_request_id, workload_id=None)
0130         assert_equal(len(req_trans_colls.keys()), 1)
0131         req_id = list(req_trans_colls.keys())[0]
0132         assert_equal(origin_request_id, req_id)
0133         assert_equal(len(req_trans_colls[req_id].keys()), 1)
0134         trans_id = list(req_trans_colls[req_id].keys())[0]
0135         assert_equal(trans_id, origin_trans_id)
0136         colls = req_trans_colls[req_id][trans_id]
0137         assert_equal(len(colls), 1)
0138         coll = colls[0]
0139         for key in coll_properties:
0140             if key == 'relation_type':
0141                 assert_equal(coll[key], CollectionRelationType.Input)
0142             if key == 'name':
0143                 assert_equal(coll[key], coll_input_properties[key])
0144
0145         req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_output_properties['name'], request_id=None, workload_id=req_properties['workload_id'])
0146         assert_equal(len(req_trans_colls.keys()), 1)
0147         req_id = list(req_trans_colls.keys())[0]
0148         assert_equal(None, req_id)
0149         assert_equal(len(req_trans_colls[req_id].keys()), 1)
0150         trans_id = list(req_trans_colls[req_id].keys())[0]
0151         assert_equal(trans_id, origin_trans_id)
0152         colls = req_trans_colls[req_id][trans_id]
0153         assert_equal(len(colls), 1)
0154         coll = colls[0]
0155         for key in coll_properties:
0156             if key == 'relation_type':
0157                 assert_equal(coll[key], CollectionRelationType.Output)
0158             if key == 'name':
0159                 assert_equal(coll[key], coll_output_properties[key])
0160
0161         req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_log_properties['name'], request_id=origin_request_id, workload_id=req_properties['workload_id'])
0162         assert_equal(len(req_trans_colls.keys()), 1)
0163         req_id = list(req_trans_colls.keys())[0]
0164         assert_equal(origin_request_id, req_id)
0165         assert_equal(len(req_trans_colls[req_id].keys()), 1)
0166         trans_id = list(req_trans_colls[req_id].keys())[0]
0167         assert_equal(trans_id, origin_trans_id)
0168         colls = req_trans_colls[req_id][trans_id]
0169         assert_equal(len(colls), 1)
0170         coll = colls[0]
0171         for key in coll_properties:
0172             if key == 'relation_type':
0173                 assert_equal(coll[key], CollectionRelationType.Log)
0174             if key == 'name':
0175                 assert_equal(coll[key], coll_log_properties[key])
0176
0177         trans_properties1 = copy.deepcopy(trans_properties)
0178         origin_trans_id1 = add_transform(**trans_properties1)
0179         coll_properties1 = copy.deepcopy(coll_properties)
0180         coll_properties1['name'] = coll_properties1['name'] + '_1'
0181         coll_properties1['transform_id'] = origin_trans_id1
0182         coll_input_properties1 = copy.deepcopy(coll_properties1)
0183         coll_input_properties1['name'] = coll_input_properties1['name'] + '_input'
0184         coll_output_properties1 = copy.deepcopy(coll_properties1)
0185         coll_output_properties1['relation_type'] = CollectionRelationType.Output
0186         coll_output_properties1['name'] = coll_output_properties1['name'] + '_output'
0187         coll_log_properties1 = copy.deepcopy(coll_properties1)
0188         coll_log_properties1['relation_type'] = CollectionRelationType.Log
0189         coll_log_properties1['name'] = coll_log_properties1['name'] + '_log'
0190
0191         origin_coll_input_id1 = add_collection(**coll_input_properties1)
0192         origin_coll_output_id1 = add_collection(**coll_output_properties1)
0193         origin_coll_log_id1 = add_collection(**coll_log_properties1)
0194         origin_coll_id_list1 = [origin_coll_input_id1, origin_coll_output_id1, origin_coll_log_id1]
0195
0196         req_trans_colls = get_collections(request_id=origin_request_id, workload_id=None)
0197         assert_equal(len(req_trans_colls.keys()), 1)
0198         req_id = list(req_trans_colls.keys())[0]
0199         assert_equal(origin_request_id, req_id)
0200         assert_equal(len(req_trans_colls[req_id].keys()), 2)
0201         for trans_id in req_trans_colls[req_id]:
0202             if trans_id == origin_trans_id:
0203                 colls = req_trans_colls[req_id][trans_id]
0204                 assert_equal(len(colls), 3)
0205                 for coll in colls:
0206                     assert_in(coll['coll_id'], origin_coll_id_list)
0207                     for key in coll_properties:
0208                         if key == 'relation_type':
0209                             if coll['coll_id'] == origin_coll_input_id:
0210                                 assert_equal(coll[key], CollectionRelationType.Input)
0211                             elif coll['coll_id'] == origin_coll_output_id:
0212                                 assert_equal(coll[key], CollectionRelationType.Output)
0213                             else:
0214                                 assert_equal(coll['coll_id'], origin_coll_log_id)
0215                                 assert_equal(coll[key], CollectionRelationType.Log)
0216                         if key == 'name':
0217                             if coll['coll_id'] == origin_coll_input_id:
0218                                 assert_equal(coll[key], coll_input_properties[key])
0219                             elif coll['coll_id'] == origin_coll_output_id:
0220                                 assert_equal(coll[key], coll_output_properties[key])
0221                             else:
0222                                 assert_equal(coll[key], coll_log_properties[key])
0223             if trans_id == origin_trans_id1:
0224                 colls = req_trans_colls[req_id][trans_id]
0225                 assert_equal(len(colls), 3)
0226                 for coll in colls:
0227                     assert_in(coll['coll_id'], origin_coll_id_list1)
0228                     for key in coll_properties:
0229                         if key == 'relation_type':
0230                             if coll['coll_id'] == origin_coll_input_id1:
0231                                 assert_equal(coll[key], CollectionRelationType.Input)
0232                             elif coll['coll_id'] == origin_coll_output_id1:
0233                                 assert_equal(coll[key], CollectionRelationType.Output)
0234                             else:
0235                                 assert_equal(coll['coll_id'], origin_coll_log_id1)
0236                                 assert_equal(coll[key], CollectionRelationType.Log)
0237                         if key == 'name':
0238                             if coll['coll_id'] == origin_coll_input_id1:
0239                                 assert_equal(coll[key], coll_input_properties1[key])
0240                             elif coll['coll_id'] == origin_coll_output_id1:
0241                                 assert_equal(coll[key], coll_output_properties1[key])
0242                             else:
0243                                 assert_equal(coll[key], coll_log_properties1[key])
0244
0245         req_trans_colls1 = get_collections(request_id=None, workload_id=req_properties['workload_id'])
0246         assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1, allow_request_id_None=True), True)
0247
0248         req_trans_colls1 = get_collections(request_id=origin_request_id, workload_id=req_properties['workload_id'])
0249         assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1), True)
0250
0251         req_trans_colls = get_collections(scope=coll_properties['scope'], name=coll_properties['name'] + "*", request_id=None, workload_id=None)
0252         assert_equal(len(req_trans_colls.keys()), 1)
0253         req_id = list(req_trans_colls.keys())[0]
0254         assert_equal(None, req_id)
0255         assert_equal(len(req_trans_colls[req_id].keys()), 2)
0256         for trans_id in req_trans_colls[req_id]:
0257             if trans_id == origin_trans_id:
0258                 colls = req_trans_colls[req_id][trans_id]
0259                 assert_equal(len(colls), 3)
0260                 for coll in colls:
0261                     assert_in(coll['coll_id'], origin_coll_id_list)
0262                     for key in coll_properties:
0263                         if key == 'relation_type':
0264                             if coll['coll_id'] == origin_coll_input_id:
0265                                 assert_equal(coll[key], CollectionRelationType.Input)
0266                             elif coll['coll_id'] == origin_coll_output_id:
0267                                 assert_equal(coll[key], CollectionRelationType.Output)
0268                             else:
0269                                 assert_equal(coll['coll_id'], origin_coll_log_id)
0270                                 assert_equal(coll[key], CollectionRelationType.Log)
0271                         if key == 'name':
0272                             if coll['coll_id'] == origin_coll_input_id:
0273                                 assert_equal(coll[key], coll_input_properties[key])
0274                             elif coll['coll_id'] == origin_coll_output_id:
0275                                 assert_equal(coll[key], coll_output_properties[key])
0276                             else:
0277                                 assert_equal(coll[key], coll_log_properties[key])
0278             if trans_id == origin_trans_id1:
0279                 colls = req_trans_colls[req_id][trans_id]
0280                 assert_equal(len(colls), 3)
0281                 for coll in colls:
0282                     assert_in(coll['coll_id'], origin_coll_id_list1)
0283                     for key in coll_properties:
0284                         if key == 'relation_type':
0285                             if coll['coll_id'] == origin_coll_input_id1:
0286                                 assert_equal(coll[key], CollectionRelationType.Input)
0287                             elif coll['coll_id'] == origin_coll_output_id1:
0288                                 assert_equal(coll[key], CollectionRelationType.Output)
0289                             else:
0290                                 assert_equal(coll['coll_id'], origin_coll_log_id1)
0291                                 assert_equal(coll[key], CollectionRelationType.Log)
0292                         if key == 'name':
0293                             if coll['coll_id'] == origin_coll_input_id1:
0294                                 assert_equal(coll[key], coll_input_properties1[key])
0295                             elif coll['coll_id'] == origin_coll_output_id1:
0296                                 assert_equal(coll[key], coll_output_properties1[key])
0297                             else:
0298                                 assert_equal(coll[key], coll_log_properties1[key])
0299
0300         req_properties = get_request_properties()
0301         origin_request_id = add_request(**req_properties)
0302
0303         trans_properties = get_transform_properties()
0304         trans_properties['request_id'] = origin_request_id
0305         origin_trans_id = add_transform(**trans_properties)
0306
0307         coll_properties = get_collection_properties()
0308         coll_properties['transform_id'] = origin_trans_id
0309         coll_input_properties = copy.deepcopy(coll_properties)
0310         coll_input_properties['name'] = coll_input_properties['name'] + '_input'
0311         coll_output_properties = copy.deepcopy(coll_properties)
0312         coll_output_properties['relation_type'] = CollectionRelationType.Output
0313         coll_output_properties['name'] = coll_output_properties['name'] + '_output'
0314
0315         origin_coll_input_id = add_collection(**coll_input_properties)
0316         origin_coll_output_id = add_collection(**coll_output_properties)
0317
0318         content_input_properties = get_content_properties()
0319         content_input_properties['coll_id'] = origin_coll_input_id
0320         origin_content_input_id_0_100 = add_content(**content_input_properties)
0321         content_input_properties_100_200 = copy.deepcopy(content_input_properties)
0322         content_input_properties_100_200['min_id'] = 100
0323         content_input_properties_100_200['max_id'] = 200
0324         origin_content_input_id_100_200 = add_content(**content_input_properties_100_200)
0325         content_input_properties_name1 = copy.deepcopy(content_input_properties)
0326         content_input_properties_name1['name'] = content_input_properties_name1['name'] + '_1'
0327         content_input_properties_name1_id = add_content(**content_input_properties_name1)
0328
0329         content_output_properties = get_content_properties()
0330         content_output_properties['content_type'] = ContentType.File
0331         content_output_properties['min_id'] = 0
0332         content_output_properties['max_id'] = 1000
0333         content_output_properties['coll_id'] = origin_coll_output_id
0334         origin_content_output_id_0_1000 = add_content(**content_output_properties)
0335         content_output_properties_0_100 = copy.deepcopy(content_output_properties)
0336         content_output_properties_0_100['min_id'] = 0
0337         content_output_properties_0_100['max_id'] = 100
0338         content_output_properties['content_type'] = ContentType.Event
0339         origin_content_output_id_0_100 = add_content(**content_output_properties_0_100)
0340         content_output_properties_100_200 = copy.deepcopy(content_output_properties)
0341         content_output_properties_100_200['min_id'] = 100
0342         content_output_properties_100_200['max_id'] = 200
0343         content_output_properties['content_type'] = ContentType.Event
0344         origin_content_output_id_100_200 = add_content(**content_output_properties_100_200)
0345         content_output_properties_name1 = copy.deepcopy(content_output_properties)
0346         content_output_properties_name1['name'] = content_output_properties_name1['name'] + '_1'
0347         content_output_properties_name1_id = add_content(**content_output_properties_name1)
0348
0349         req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'] + "*",
0350                                                request_id=origin_request_id, workload_id=req_properties['workload_id'])
0351         coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
0352         coll_input_scope_name = '%s:%s' % (coll_input_properties['scope'], coll_input_properties['name'])
0353         coll_output_scope_name = '%s:%s' % (coll_output_properties['scope'], coll_output_properties['name'])
0354         coll_scope_names = [scope_name for scope_name in coll_contents]
0355         assert_equal(coll_scope_names, [coll_input_scope_name, coll_output_scope_name])
0356         input_contents = coll_contents[coll_input_scope_name]['contents']
0357         output_contents = coll_contents[coll_output_scope_name]['contents']
0358         assert_equal(len(input_contents), 3)
0359         assert_equal(len(output_contents), 4)
0360         input_content_ids = [input_content['content_id'] for input_content in input_contents]
0361         assert_equal(input_content_ids, [origin_content_input_id_0_100, origin_content_input_id_100_200, content_input_properties_name1_id])
0362         output_content_ids = [output_content['content_id'] for output_content in output_contents]
0363         output_content_ids.sort()
0364         assert_equal(output_content_ids, [origin_content_output_id_0_1000, origin_content_output_id_0_100,
0365                                           origin_content_output_id_100_200, content_output_properties_name1_id])
0366
0367         req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'] + "*",
0368                                                request_id=origin_request_id, workload_id=req_properties['workload_id'],
0369                                                relation_type=CollectionRelationType.Input)
0370         coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
0371         coll_input_scope_name = '%s:%s' % (coll_input_properties['scope'], coll_input_properties['name'])
0372         coll_scope_names = list(coll_contents.keys())
0373         assert_equal(coll_scope_names, [coll_input_scope_name])
0374
0375         req_trans_coll_contents = get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'] + "*",
0376                                                request_id=origin_request_id, workload_id=req_properties['workload_id'],
0377                                                relation_type=CollectionRelationType.Output)
0378         coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
0379         coll_output_scope_name = '%s:%s' % (coll_output_properties['scope'], coll_output_properties['name'])
0380         coll_scope_names = list(coll_contents.keys())
0381         assert_equal(coll_scope_names, [coll_output_scope_name])
0382
0383         contents = [{'scope': content_output_properties['scope'], 'name': content_output_properties['name'],
0384                      'min_id': content_output_properties['min_id'], 'max_id': content_output_properties['max_id'],
0385                      'status': ContentStatus.Available, 'path': '/abc/test_path'},
0386                     {'scope': content_output_properties_name1['scope'], 'name': content_output_properties_name1['name'],
0387                      'min_id': content_output_properties_name1['min_id'], 'max_id': content_output_properties_name1['max_id'],
0388                      'status': ContentStatus.Failed}]
0389         register_output_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
0390                                  contents=contents, request_id=origin_request_id, workload_id=None)
0391         content = get_content(content_id=origin_content_output_id_0_1000)
0392         assert_equal(content['status'], ContentStatus.Available)
0393         assert_equal(content['path'], '/abc/test_path')
0394         content = get_content(content_id=content_output_properties_name1_id)
0395         assert_equal(content['status'], ContentStatus.Failed)
0396
0397         contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
0398                                       scope=content_output_properties['scope'], name=content_output_properties['name'],
0399                                       min_id=None, max_id=None, request_id=origin_request_id,
0400                                       workload_id=req_properties['workload_id'], only_return_best_match=False)
0401         assert_equal(len(contents), 3)
0402         content_ids = [content['content_id'] for content in contents]
0403         content_ids.sort()
0404         content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100, origin_content_output_id_100_200]
0405         content_ids1.sort()
0406         assert_equal(content_ids, content_ids1)
0407
0408         contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
0409                                       scope=content_output_properties['scope'], name=content_output_properties['name'],
0410                                       min_id=0, max_id=50, request_id=origin_request_id,
0411                                       workload_id=req_properties['workload_id'], only_return_best_match=False)
0412         assert_equal(len(contents), 2)
0413         content_ids = [content['content_id'] for content in contents]
0414         content_ids.sort()
0415         content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100]
0416         content_ids1.sort()
0417         assert_equal(content_ids, content_ids1)
0418
0419         contents = get_match_contents(coll_scope=coll_output_properties['scope'], coll_name=coll_output_properties['name'],
0420                                       scope=content_output_properties['scope'], name=content_output_properties['name'],
0421                                       min_id=0, max_id=50, request_id=origin_request_id,
0422                                       workload_id=req_properties['workload_id'], only_return_best_match=True)
0423         assert_equal(len(contents), 1)
0424         content_ids = [content['content_id'] for content in contents]
0425         assert_equal(content_ids, [origin_content_output_id_0_100])
0426
0427     @unittest.skipIf(not has_config(), "No config file")
0428     @unittest.skipIf(not check_user_proxy(), "No user proxy to access REST")
0429     @unittest.skipIf(not check_rest_host(), "REST host is not defined")
0430     def test_catalog_rest(self):
0431         """ Catalog (Rest): Test catalog rest functions """
0432         host = get_rest_host()
0433         client = Client(host=host)
0434
0435         req_properties = get_request_properties()
0436         origin_request_id = add_request(**req_properties)
0437
0438         trans_properties = get_transform_properties()
0439         trans_properties['request_id'] = origin_request_id
0440         origin_trans_id = add_transform(**trans_properties)
0441
0442         coll_properties = get_collection_properties()
0443         coll_properties['transform_id'] = origin_trans_id
0444         coll_properties['relation_type'] = CollectionRelationType.Output
0445         origin_coll_id = add_collection(**coll_properties)
0446
0447         req_trans_colls = client.get_collections(request_id=origin_request_id, workload_id=None)
0448         assert_equal(len(req_trans_colls.keys()), 1)
0449         req_id = list(req_trans_colls.keys())[0]
0450         assert_equal(origin_request_id, req_id)
0451         assert_equal(len(req_trans_colls[req_id].keys()), 1)
0452         trans_id = list(req_trans_colls[req_id].keys())[0]
0453         assert_equal(trans_id, origin_trans_id)
0454         colls = req_trans_colls[req_id][trans_id]
0455         assert_equal(len(colls), 1)
0456         assert_equal(colls[0]['coll_id'], origin_coll_id)
0457
0458         req_trans_colls1 = client.get_collections(request_id=None, workload_id=req_properties['workload_id'])
0459         assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1, allow_request_id_None=True), True)
0460
0461         req_trans_colls1 = client.get_collections(scope=coll_properties['scope'], name=coll_properties['name'],
0462                                                   request_id=None, workload_id=None)
0463         assert_equal(is_same_req_trans_colls(req_trans_colls, req_trans_colls1, allow_request_id_None=True), True)
0464
0465         content_output_properties = get_content_properties()
0466         content_output_properties['content_type'] = ContentType.File
0467         content_output_properties['min_id'] = 0
0468         content_output_properties['max_id'] = 1000
0469         content_output_properties['coll_id'] = origin_coll_id
0470         origin_content_output_id_0_1000 = add_content(**content_output_properties)
0471         content_output_properties_0_100 = copy.deepcopy(content_output_properties)
0472         content_output_properties_0_100['min_id'] = 0
0473         content_output_properties_0_100['max_id'] = 100
0474         content_output_properties['content_type'] = ContentType.Event
0475         origin_content_output_id_0_100 = add_content(**content_output_properties_0_100)
0476         content_output_properties_100_200 = copy.deepcopy(content_output_properties)
0477         content_output_properties_100_200['min_id'] = 100
0478         content_output_properties_100_200['max_id'] = 200
0479         content_output_properties['content_type'] = ContentType.Event
0480         origin_content_output_id_100_200 = add_content(**content_output_properties_100_200)
0481         content_output_properties_name1 = copy.deepcopy(content_output_properties)
0482         content_output_properties_name1['name'] = content_output_properties_name1['name'] + '_1'
0483         content_output_properties_name1_id = add_content(**content_output_properties_name1)
0484
0485         req_trans_coll_contents = client.get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
0486                                                       request_id=origin_request_id, workload_id=req_properties['workload_id'])
0487         coll_contents = req_trans_coll_contents[origin_request_id][origin_trans_id]
0488         coll_scope_name = '%s:%s' % (coll_properties['scope'], coll_properties['name'])
0489         coll_scope_names = [scope_name for scope_name in coll_contents]
0490         assert_equal(coll_scope_names, [coll_scope_name])
0491         contents = coll_contents[coll_scope_name]['contents']
0492         assert_equal(len(contents), 4)
0493         output_content_ids = [output_content['content_id'] for output_content in contents]
0494         output_content_ids.sort()
0495         assert_equal(output_content_ids, [origin_content_output_id_0_1000, origin_content_output_id_0_100,
0496                                           origin_content_output_id_100_200, content_output_properties_name1_id])
0497
0498         req_trans_coll_contents1 = client.get_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
0499                                                        request_id=origin_request_id, workload_id=req_properties['workload_id'],
0500                                                        relation_type=CollectionRelationType.Output)
0501         assert_equal(is_same_req_trans_coll_contents(req_trans_coll_contents, req_trans_coll_contents1), True)
0502
0503         contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
0504                                              scope=content_output_properties['scope'], name=content_output_properties['name'],
0505                                              min_id=None, max_id=None, request_id=origin_request_id,
0506                                              workload_id=req_properties['workload_id'], only_return_best_match=False)
0507         assert_equal(len(contents), 3)
0508         content_ids = [content['content_id'] for content in contents]
0509         content_ids.sort()
0510         content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100, origin_content_output_id_100_200]
0511         content_ids1.sort()
0512         assert_equal(content_ids, content_ids1)
0513
0514         contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
0515                                              scope=content_output_properties['scope'], name=content_output_properties['name'],
0516                                              min_id=0, max_id=50, request_id=origin_request_id,
0517                                              workload_id=req_properties['workload_id'], only_return_best_match=False)
0518         assert_equal(len(contents), 2)
0519         content_ids = [content['content_id'] for content in contents]
0520         content_ids.sort()
0521         content_ids1 = [origin_content_output_id_0_1000, origin_content_output_id_0_100]
0522         content_ids1.sort()
0523         assert_equal(content_ids, content_ids1)
0524
0525         contents = client.get_match_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
0526                                              scope=content_output_properties['scope'], name=content_output_properties['name'],
0527                                              min_id=0, max_id=50, request_id=origin_request_id,
0528                                              workload_id=req_properties['workload_id'], only_return_best_match=True)
0529         assert_equal(len(contents), 1)
0530         content_ids = [content['content_id'] for content in contents]
0531         assert_equal(content_ids, [origin_content_output_id_0_100])
0532
0533         contents = [{'scope': content_output_properties['scope'], 'name': content_output_properties['name'],
0534                      'min_id': content_output_properties['min_id'], 'max_id': content_output_properties['max_id'],
0535                      'status': ContentStatus.Available, 'path': '/abc/test_path'},
0536                     {'scope': content_output_properties_name1['scope'], 'name': content_output_properties_name1['name'],
0537                      'min_id': content_output_properties_name1['min_id'], 'max_id': content_output_properties_name1['max_id'],
0538                      'status': ContentStatus.Failed}]
0539         client.register_contents(coll_scope=coll_properties['scope'], coll_name=coll_properties['name'],
0540                                  contents=contents, request_id=origin_request_id,
0541                                  workload_id=req_properties['workload_id'])
0542         content = get_content(content_id=origin_content_output_id_0_1000)
0543         assert_equal(content['status'], ContentStatus.Available)
0544         assert_equal(content['path'], '/abc/test_path')
0545         content = get_content(content_id=content_output_properties_name1_id)
0546         assert_equal(content['status'], ContentStatus.Failed)