idds/tests/test_big_request.py

0001 import logging
0002 import sys
0003 import datetime
0004
0005
0006 from idds.common.utils import setup_logging, json_dumps                 # noqa F401
0007 setup_logging(__name__, stream=sys.stdout, loglevel=logging.DEBUG)
0008
0009 from idds.common.constants import ContentStatus, ContentType, ContentRelationType, ContentLocking          # noqa F401
0010 from idds.core.requests import get_requests              # noqa F401
0011 from idds.core.messages import retrieve_messages         # noqa F401
0012 from idds.core.transforms import get_transforms          # noqa F401
0013 from idds.core.workprogress import get_workprogresses    # noqa F401
0014 from idds.core.processings import get_processings        # noqa F401
0015 from idds.core import transforms as core_transforms      # noqa F401
0016 from idds.orm.contents import get_input_contents         # noqa F401
0017 from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old     # noqa F401
0018
0019
0020 def show_works(req):
0021     workflow = req['processing_metadata']['workflow']
0022     print(workflow.independent_works)
0023     print(len(workflow.independent_works))
0024     print(workflow.works_template.keys())
0025     print(len(workflow.works_template.keys()))
0026     print(workflow.work_sequence.keys())
0027     print(len(workflow.work_sequence.keys()))
0028     print(workflow.works.keys())
0029     print(len(workflow.works.keys()))
0030
0031     work_ids = []
0032     for i_id in workflow.works:
0033         work = workflow.works[i_id]
0034         print(i_id)
0035         print(work.work_name)
0036         print(work.task_name)
0037         print(work.work_id)
0038         work_ids.append(work.work_id)
0039     print(work_ids)
0040
0041
0042 print("start at: ", datetime.datetime.utcnow())
0043 # 283511, 283517
0044 # reqs = get_requests(request_id=599, with_detail=True, with_metadata=True)
0045 # reqs = get_requests(request_id=283511, with_request=True, with_detail=False, with_metadata=True)
0046 # reqs = get_requests(request_id=298163, with_request=True, with_detail=False, with_metadata=True)
0047 # reqs = get_requests(request_id=298557, with_request=True, with_detail=False, with_metadata=True)
0048 # reqs = get_requests(request_id=299111, with_request=True, with_detail=False, with_metadata=True)
0049 # reqs = get_requests(request_id=299235, with_request=True, with_detail=False, with_metadata=True)
0050 # reqs = get_requests(request_id=965, with_request=True, with_detail=False, with_metadata=True)
0051 reqs = get_requests(request_id=274, with_request=True, with_detail=False, with_metadata=True)
0052 print("got requests at: ", datetime.datetime.utcnow())
0053 for req in reqs:
0054     # print(req['request_id'])
0055     # print(rets)
0056     # print(json_dumps(req, sort_keys=True, indent=4))
0057     # show_works(req)
0058     pass
0059     workflow = req['request_metadata']['workflow']
0060
0061     print("generating works at: ", datetime.datetime.utcnow())
0062     works = workflow.get_new_works()
0063     print("generated works at: ", datetime.datetime.utcnow())
0064
0065     for work in works:
0066         new_work = work
0067         new_work.add_proxy(workflow.get_proxy())
0068 print("end at: ", datetime.datetime.utcnow())