pandaharvester/harvesterworkermaker/dummy_dynamic_worker_maker.py

0001 import random
0002
0003 from pandaharvester.harvestercore.work_spec import WorkSpec
0004
0005 from .base_worker_maker import BaseWorkerMaker
0006
0007 # dummy worker maker
0008
0009
0010 class DummyDynamicWorkerMaker(BaseWorkerMaker):
0011     # constructor
0012     def __init__(self, **kwarg):
0013         BaseWorkerMaker.__init__(self, **kwarg)
0014
0015     # make a worker from jobs
0016     def make_worker(self, jobspec_list, queue_config, job_type, resource_type, **kwargs):
0017         workSpec = WorkSpec()
0018         workSpec.resourceType = resource_type
0019         if len(jobspec_list) > 0:
0020             workSpec.nCore = 0
0021             workSpec.minRamCount = 0
0022             workSpec.maxDiskCount = 0
0023             workSpec.maxWalltime = 0
0024             for jobSpec in jobspec_list:
0025                 try:
0026                     workSpec.nCore += jobSpec.jobParams["coreCount"]
0027                 except Exception:
0028                     workSpec.nCore += 1
0029                 try:
0030                     workSpec.minRamCount += jobSpec.jobParams["minRamCount"]
0031                 except Exception:
0032                     pass
0033                 try:
0034                     workSpec.maxDiskCount += jobSpec.jobParams["maxDiskCount"]
0035                 except Exception:
0036                     pass
0037                 try:
0038                     if jobSpec.jobParams["maxWalltime"] not in (None, "NULL"):
0039                         workSpec.maxWalltime = max(int(queue_config.walltimeLimit), jobSpec.jobParams["maxWalltime"])
0040                     else:
0041                         workSpec.maxWalltime = queue_config.walltimeLimit
0042                 except Exception:
0043                     pass
0044         return workSpec
0045
0046     # get number of jobs per worker based on dynamic information such as # of free CPUs at that time.
0047     # N.B. n_worker is the number of available slots which may be useful for some workflow
0048     def get_num_jobs_per_worker(self, n_workers):
0049         # dummy. should use batch system info, etc
0050         return random.randint(self.minJobsPerWorker, self.maxJobsPerWorker)
0051
0052     # get number of workers per job based on dynamic information
0053     def get_num_workers_per_job(self, n_workers):
0054         # dummy. should use batch system info, etc
0055         return random.randint(self.minWorkersPerJob, self.maxWorkersPerJob)