idds/tests/catalog_test.py

0001 #!/usr/bin/env python
0002 #
0003 # Licensed under the Apache License, Version 2.0 (the "License");
0004 # You may not use this file except in compliance with the License.
0005 # You may obtain a copy of the License at
0006 # http://www.apache.org/licenses/LICENSE-2.0OA
0007 #
0008 # Authors:
0009 # - Wen Guan, <wen.guan@cern.ch>, 2020
0010
0011
0012 """
0013 Test hyper parameter optimization test  client.
0014 """
0015
0016 from idds.client.client import Client
0017 from idds.common.constants import CollectionRelationType, ContentStatus
0018 from idds.common.utils import get_rest_host
0019
0020
0021 # host = "https://aipanda181.cern.ch:443/idds"
0022 host = get_rest_host()
0023
0024 client = Client(host=host)
0025 # props['request_metadata']['result_parser'] = 'default'
0026
0027 scope = 'data16_13TeV'
0028 name = 'data16_13TeV.00298862.physics_Main.daq.RAW.idds.stagein'
0029 request_id = 12
0030 workload_id = 1601235010
0031 relation_type = CollectionRelationType.Output  # Input, Log
0032 status = ContentStatus.Available  # New, Processing, Available, ...
0033
0034 colls = client.get_collections(scope=scope, name=name, request_id=request_id, workload_id=workload_id, relation_type=relation_type)
0035 print(colls)
0036 # example outputs
0037 # [{'relation_type': <CollectionRelationType.Output: 1>, 'next_poll_at': 'Sun, 27 Sep 2020 19:30:16 UTC', 'storage_id': None, 'scope': 'data16_13TeV', 'accessed_at': 'Sun, 27 Sep 2020 21:31:40 UTC', 'new_files': 0, 'name': 'data16_13TeV.00298862.physics_Main.daq.RAW.idds.stagein', 'expired_at': 'Tue, 27 Oct 2020 19:30:10 UTC', 'processed_files': 0, 'bytes': 0, 'coll_metadata': {'internal_id': 'db1bb0dc-00f7-11eb-a5d2-fa163eb98fd2'}, 'request_id': 12, 'processing_files': 0, 'status': <CollectionStatus.Open: 3>, 'coll_id': 16, 'processing_id': None, 'substatus': None, 'workload_id': 1601235010, 'retries': 0, 'locking': <CollectionLocking.Idle: 0>, 'coll_type': <CollectionType.Dataset: 1>, 'created_at': 'Sun, 27 Sep 2020 19:30:16 UTC', 'total_files': 0, 'transform_id': 9, 'updated_at': 'Sun, 27 Sep 2020 21:31:40 UTC'}]  # noqa: E501
0038
0039 contents = client.get_contents(coll_scope=scope, coll_name=name, request_id=request_id, workload_id=workload_id, relation_type=relation_type, status=status)
0040 for content in contents:
0041     # print(content)
0042     pass
0043
0044 # example outputs
0045 # {'substatus': <ContentStatus.Available: 2>, 'transform_id': 9, 'path': None, 'locking': <ContentLocking.Idle: 0>, 'map_id': 8895, 'created_at': 'Sun, 27 Sep 2020 19:30:40 UTC', 'bytes': 534484132, 'scope': 'data16_13TeV', 'updated_at': 'Sun, 27 Sep 2020 19:44:55 UTC', 'md5': None, 'name': 'data16_13TeV.00298862.physics_Main.daq.RAW._lb0538._SFO-5._0003.data', 'accessed_at': 'Sun, 27 Sep 2020 19:44:56 UTC', 'adler32': 'e8543989', 'content_id': 2361650, 'min_id': 0, 'expired_at': 'Tue, 27 Oct 2020 19:30:26 UTC', 'coll_id': 16, 'processing_id': None, 'max_id': 579, 'content_metadata': {'events': 579}, 'storage_id': None, 'request_id': 12, 'content_type': <ContentType.File: 0>, 'workload_id': 1601235010, 'retries': 0, 'status': <ContentStatus.Available: 2>}  # noqa: E501