idds/tests/test_transform_collection_content.py

0001 #!/usr/bin/env python
0002 #
0003 # Licensed under the Apache License, Version 2.0 (the "License");
0004 # You may not use this file except in compliance with the License.
0005 # You may obtain a copy of the License at
0006 # http://www.apache.org/licenses/LICENSE-2.0OA
0007 #
0008 # Authors:
0009 # - Wen Guan, <wen.guan@cern.ch>, 2019
0010
0011
0012 """
0013 Test Request.
0014 """
0015
0016 import copy
0017
0018 import unittest2 as unittest
0019 from nose.tools import assert_equal, assert_in
0020
0021 from idds.common.constants import (TransformStatus, CollectionStatus, ContentStatus, ContentType)
0022 from idds.common.utils import check_database, has_config, setup_logging
0023 from idds.orm.requests import add_request
0024 from idds.orm.transforms import (add_transform, delete_transform,
0025                                  update_transform, get_transform)
0026 from idds.orm.collections import (add_collection, get_collection_id,
0027                                   get_collection, update_collection,
0028                                   delete_collection, get_collections,
0029                                   get_collection_ids_by_transform_id)
0030 from idds.orm.contents import (add_content, get_content, update_content,
0031                                delete_content, get_contents,
0032                                get_match_contents, update_contents)
0033 from idds.tests.common import (get_request_properties, get_transform_properties,
0034                                get_collection_properties, get_content_properties)
0035
0036 setup_logging(__name__)
0037
0038
0039 class TestTransformCollectionContent(unittest.TestCase):
0040
0041     @unittest.skipIf(not has_config(), "No config file")
0042     @unittest.skipIf(not check_database(), "Database is not defined")
0043     def test_create_and_check_for_transform_orm(self):
0044         """ Transform (ORM): Test the creation, query, and cancel of a Transform """
0045         trans_properties = get_transform_properties()
0046
0047         trans_id = add_transform(**trans_properties)
0048         transform = get_transform(transform_id=trans_id)
0049         for key in trans_properties:
0050             assert_equal(transform[key], trans_properties[key])
0051
0052         update_transform(trans_id, {'status': TransformStatus.Failed})
0053         transform = get_transform(transform_id=trans_id)
0054         assert_equal(transform['status'], TransformStatus.Failed)
0055
0056         delete_transform(trans_id)
0057
0058         t = get_transform(transform_id=trans_id)
0059         assert_equal(t, None)
0060
0061     def test_create_and_check_for_transform_collection_content_orm(self):
0062         """ Transform/Collection/Content (ORM): Test the creation, query, and cancel of a Transform/Collection/Content """
0063
0064         trans_properties = get_transform_properties()
0065         coll_properties = get_collection_properties()
0066         content_properties = get_content_properties()
0067
0068         trans_id = add_transform(**trans_properties)
0069
0070         coll_properties['transform_id'] = trans_id
0071         coll_id = add_collection(**coll_properties)
0072         coll_id_1 = get_collection_id(transform_id=trans_id, relation_type=coll_properties['relation_type'])
0073         assert_equal(coll_id, coll_id_1)
0074
0075         coll = get_collection(coll_id=coll_id)
0076         for key in coll_properties:
0077             assert_equal(coll[key], coll_properties[key])
0078
0079         update_collection(coll_id, {'status': CollectionStatus.Closed})
0080         coll = get_collection(coll_id=coll_id)
0081         assert_equal(coll['status'], CollectionStatus.Closed)
0082
0083         content_properties['coll_id'] = coll_id
0084         content_id = add_content(**content_properties)
0085         content_1 = get_content(coll_id=coll_id, scope=content_properties['scope'],
0086                                 name=content_properties['name'],
0087                                 content_type=ContentType.File)
0088         content_id_1 = content_1['content_id']
0089         assert_equal(content_id, content_id_1)
0090
0091         content = get_content(content_id=content_id)
0092         update_content(content_id=content_id, parameters={'status': ContentStatus.Lost})
0093         content = get_content(content_id=content_id)
0094         assert_equal(content['status'], ContentStatus.Lost)
0095
0096         delete_content(content_id=content_id)
0097         c = get_content(content_id=content_id)
0098         assert_equal(c, None)
0099
0100         delete_collection(coll_id=coll_id)
0101         coll = get_collection(coll_id=coll_id)
0102         assert_equal(coll, None)
0103
0104         delete_transform(transform_id=trans_id)
0105         t = get_transform(transform_id=trans_id)
0106         assert_equal(t, None)
0107
0108     def test_get_collections_orm(self):
0109         """ Collections (ORM): Test get collections """
0110
0111         req_properties = get_request_properties()
0112         trans_properties = get_transform_properties()
0113         coll_properties = get_collection_properties()
0114
0115         request_id = add_request(**req_properties)
0116         trans_properties['request_id'] = request_id
0117
0118         trans_id = add_transform(**trans_properties)
0119
0120         coll_properties['transform_id'] = trans_id
0121         origin_coll_id = add_collection(**coll_properties)
0122         coll_properties1 = copy.deepcopy(coll_properties)
0123         coll_properties1['name'] = coll_properties['name'] + '_1'
0124         origin_coll_id1 = add_collection(**coll_properties1)
0125         origin_coll_id_list = [origin_coll_id, origin_coll_id1]
0126
0127         colls = get_collections(transform_id=trans_id)
0128         assert_equal(len(colls), 2)
0129         for coll in colls:
0130             assert_in(coll['coll_id'], origin_coll_id_list)
0131             for key in coll_properties:
0132                 if key == 'name':
0133                     continue
0134                 assert_equal(coll[key], coll_properties[key])
0135
0136         coll_ids = get_collection_ids_by_transform_id(transform_id=trans_id)
0137         assert_equal(len(coll_ids), 2)
0138         for coll_id in coll_ids:
0139             assert_in(coll_id, origin_coll_id_list)
0140
0141         colls = get_collections(scope=coll_properties['scope'],
0142                                 name=coll_properties['name'],
0143                                 transform_id=[trans_id])
0144         assert_equal(len(colls), 1)
0145         for coll in colls:
0146             assert_in(coll['coll_id'], origin_coll_id_list)
0147             for key in coll_properties:
0148                 if key == 'name':
0149                     continue
0150                 assert_equal(coll[key], coll_properties[key])
0151
0152     def test_contents_orm(self):
0153         """ Contents (ORM): Test contents """
0154
0155         req_properties = get_request_properties()
0156         trans_properties = get_transform_properties()
0157         coll_properties = get_collection_properties()
0158         content_properties = get_content_properties()
0159
0160         request_id = add_request(**req_properties)
0161         trans_properties['request_id'] = request_id
0162
0163         trans_id = add_transform(**trans_properties)
0164
0165         coll_properties['transform_id'] = trans_id
0166         coll_id = add_collection(**coll_properties)
0167
0168         content_properties['coll_id'] = coll_id
0169         origin_content_id = add_content(**content_properties)
0170         content_properties1 = copy.deepcopy(content_properties)
0171         content_properties1['min_id'] = 101
0172         content_properties1['max_id'] = 200
0173         origin_content_id1 = add_content(**content_properties1)
0174         content_properties2 = copy.deepcopy(content_properties)
0175         content_properties2['min_id'] = 0
0176         content_properties2['max_id'] = 200
0177         origin_content_id2 = add_content(**content_properties2)
0178         content_properties3 = copy.deepcopy(content_properties)
0179         content_properties3['name'] = content_properties3['name'] + '_1'
0180         origin_content_id3 = add_content(**content_properties3)
0181         origin_content_ids = [origin_content_id, origin_content_id1,
0182                               origin_content_id2, origin_content_id3]
0183
0184         contents = get_contents(coll_id=coll_id)
0185         assert_equal(len(contents), 4)
0186         for content in contents:
0187             assert_in(content['content_id'], origin_content_ids)
0188
0189         contents = get_contents(scope=content_properties['scope'],
0190                                 name=content_properties['name'],
0191                                 coll_id=coll_id)
0192         assert_equal(len(contents), 3)
0193         for content in contents:
0194             assert_in(content['content_id'], origin_content_ids)
0195
0196         contents = get_contents(scope=content_properties3['scope'],
0197                                 name=content_properties3['name'],
0198                                 coll_id=coll_id)
0199         assert_equal(len(contents), 1)
0200         assert_equal(contents[0]['content_id'], origin_content_id3)
0201
0202         contents = get_match_contents(coll_id=content_properties['coll_id'],
0203                                       scope=content_properties['scope'],
0204                                       name=content_properties['name'],
0205                                       min_id=content_properties['min_id'],
0206                                       max_id=content_properties['max_id'])
0207         assert_equal(len(contents), 2)
0208         for content in contents:
0209             assert_in(content['content_id'], [origin_content_id, origin_content_id2])
0210
0211         to_updates = [{'path': 'test_path1', 'status': ContentStatus.Processing, 'content_id': origin_content_id},
0212                       {'path': 'test_path2', 'status': ContentStatus.Processing, 'content_id': origin_content_id1}]
0213         update_contents(to_updates)
0214         content = get_content(content_id=origin_content_id)
0215         assert_equal(content['status'], ContentStatus.Processing)
0216         assert_equal(content['path'], 'test_path1')
0217         content = get_content(content_id=origin_content_id1)
0218         assert_equal(content['status'], ContentStatus.Processing)
0219         assert_equal(content['path'], 'test_path2')