idds/tests/find_dependencies.py

0001
0002 from idds.common.utils import json_dumps                 # noqa F401
0003 from idds.common.constants import ContentStatus, ContentType, ContentRelationType, ContentLocking          # noqa F401
0004 from idds.core.requests import get_requests              # noqa F401
0005 from idds.core.messages import retrieve_messages         # noqa F401
0006 from idds.core.transforms import get_transforms          # noqa F401
0007 from idds.core.workprogress import get_workprogresses    # noqa F401
0008 from idds.core.processings import get_processings        # noqa F401
0009 from idds.core import transforms as core_transforms      # noqa F401
0010 from idds.core import catalog as core_catalog            # noqa F401
0011 from idds.orm.contents import get_input_contents         # noqa F401
0012 from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old     # noqa F401
0013
0014
0015 def get_input_dep(contents, request_id, transform_id, coll_id, scope, name):
0016     # print(request_id, transform_id, coll_id, scope, name)
0017     map_id = None
0018     for content in contents:
0019         if content['transform_id'] == transform_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output:
0020             # print(content)
0021             print("output name: %s, status: %s" % (content['name'], content['status']))
0022             map_id = content['map_id']
0023         # elif coll_id and content['coll_id'] == coll_id and content['name'] == name and content['content_relation_type'] == ContentRelationType.Output:
0024         #    # print(content)
0025         #    map_id = content['map_id']
0026
0027     print(map_id)
0028     deps = []
0029     for content in contents:
0030         if content['transform_id'] == transform_id and content['map_id'] == map_id and content['content_relation_type'] == ContentRelationType.InputDependency:
0031             # print(content)
0032             print("Input dependency name: %s, status: %s" % (content['name'], content['status']))
0033             deps.append({'request_id': content['request_id'],
0034                          'transform_id': content['transform_id'],
0035                          'coll_id': content['coll_id'],
0036                          'scope': content['scope'],
0037                          'name': content['name']})
0038     return deps
0039
0040
0041 def get_transform_id(collections, coll_id):
0042     for coll in collections:
0043         if coll['coll_id'] == coll_id:
0044             return coll['transform_id']
0045
0046
0047 def get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=1):
0048     deps = get_input_dep(contents, request_id, transform_id, coll_id, scope, name)
0049     print("Step: %s" % step)
0050     print("(%s, %s, %s, %s, %s) depents on %s" % (request_id, transform_id, coll_id, scope, name, deps))
0051     step += 1
0052     for dep in deps:
0053         coll_id = dep['coll_id']
0054         transform_id = get_transform_id(collections, coll_id)
0055         dep['transform_id'] = transform_id
0056         get_dep_link(collections=collections, contents=contents, step=step, **dep)
0057
0058
0059 def get_dep_links(request_id, transform_id, coll_id, scope, name, step=1):
0060     collections = core_catalog.get_collections(request_id=request_id)
0061     contents = core_catalog.get_contents(request_id=request_id)
0062     get_dep_link(collections, contents, request_id, transform_id, coll_id, scope, name, step=step)
0063
0064
0065 if __name__ == '__main__':
0066     request_id = 1689
0067     transform_id = 14713
0068     coll_id = None
0069     scope = 'pseudo_dataset'
0070     name = '94248742-a255-4541-ab38-ab69364a4c88_transformPreSourceTable_23224_72+qgraphNodeId:94248742-a255-4541-ab38-ab69364a4c88+qgraphId:1657127232.749359-46373'
0071
0072     # get_input_dep(request_id, transform_id, scope, name)
0073     get_dep_links(request_id, transform_id, coll_id, scope, name)