pandaserver/taskbuffer/ResourceSpec.py

0001 """
0002 Resource type specification for JEDI
0003
0004 """
0005
0006 from . import JobUtils
0007
0008 HIMEM_THRESHOLD = 2000  # MB per core
0009 BASIC_RESOURCE_TYPE = "SCORE"
0010
0011
0012 class ResourceSpecMapper(object):
0013     def __init__(self, resource_types):
0014         """
0015         :param resource_types: list of ResourceSpec objects
0016         """
0017         self.resource_types = resource_types
0018
0019     def is_single_core(self, resource_name):
0020         for resource_type in self.resource_types:
0021             if resource_type.resource_name == resource_name:
0022                 return resource_type.is_single_core()
0023         return False
0024
0025     def is_multi_core(self, resource_name):
0026         for resource_type in self.resource_types:
0027             if resource_type.resource_name == resource_name:
0028                 return resource_type.is_multi_core()
0029         return False
0030
0031     def is_high_memory(self, resource_name, memory_threshold=HIMEM_THRESHOLD):
0032         for resource_type in self.resource_types:
0033             if resource_type.resource_name == resource_name:
0034                 if resource_type.maxrampercore is None or resource_type.maxrampercore > memory_threshold:
0035                     return True
0036         return False
0037
0038     def translate_resourcetype_to_cores(self, resource_name, cores_queue):
0039         # if the resource type is multi-core, return the number of cores in the queue
0040         if self.is_multi_core(resource_name):
0041             return cores_queue
0042
0043         return 1
0044
0045     def filter_out_high_memory_resourcetypes(self, memory_threshold=HIMEM_THRESHOLD):
0046         resource_names = list(
0047             map(
0048                 lambda resource_type: resource_type.resource_name,
0049                 filter(lambda resource_type: not self.is_high_memory(resource_type.resource_name, memory_threshold=memory_threshold), self.resource_types),
0050             )
0051         )
0052         return resource_names
0053
0054
0055 class ResourceSpec(object):
0056     # attributes
0057     attributes = (
0058         "resource_name",
0059         "mincore",
0060         "maxcore",
0061         "minrampercore",
0062         "maxrampercore",
0063     )
0064
0065     def __init__(self, resource_name, mincore, maxcore, minrampercore, maxrampercore):
0066         object.__setattr__(self, "resource_name", resource_name)
0067         object.__setattr__(self, "mincore", mincore)
0068         object.__setattr__(self, "maxcore", maxcore)
0069         object.__setattr__(self, "minrampercore", minrampercore)
0070         object.__setattr__(self, "maxrampercore", maxrampercore)
0071
0072     def match_task(self, task_spec):
0073         return self.match_task_basic(
0074             task_spec.coreCount,
0075             task_spec.ramCount,
0076             task_spec.baseRamCount,
0077             task_spec.ramUnit,
0078         )
0079
0080     def match_task_basic(self, corecount, ramcount, base_ramcount, ram_unit):
0081         # Default parameters
0082         if corecount is None:  # corecount None is also used for 1
0083             corecount = 1
0084         elif corecount == 0:  # corecount 0 means it can be anything. We will use 8 as a standard MCORE default
0085             corecount = 8
0086
0087         if ramcount is None:
0088             ramcount = 0
0089
0090         if base_ramcount is None:
0091             base_ramcount = 0
0092
0093         # check min cores
0094         if self.mincore is not None and corecount < self.mincore:
0095             return False
0096
0097         # check max cores
0098         if self.maxcore is not None and corecount > self.maxcore:
0099             return False
0100
0101         if ram_unit in ("MBPerCore", "MBPerCoreFixed"):
0102             ram_per_core = (ramcount * corecount + base_ramcount) / corecount
0103         else:
0104             ram_per_core = (ramcount + base_ramcount) / corecount
0105         ram_per_core = JobUtils.compensate_ram_count(ram_per_core)
0106
0107         # check min ram
0108         if self.minrampercore is not None and ram_per_core < self.minrampercore:
0109             return False
0110
0111         # check max ram
0112         if self.maxrampercore is not None and ram_per_core > self.maxrampercore:
0113             return False
0114
0115         return True
0116
0117     def match_job(self, job_spec):
0118         # Default parameters
0119         if job_spec.coreCount in (None, "NULL"):  # corecount None is also used for 1
0120             corecount = 1
0121         elif job_spec.coreCount == 0:  # corecount 0 means it can be anything. We will use 8 as a standard MCORE default
0122             corecount = 8
0123         else:
0124             corecount = job_spec.coreCount
0125
0126         if job_spec.minRamCount in (
0127             None,
0128             "NULL",
0129         ):  # jobs come with ram already pre-calculated and in MB
0130             ramcount = 0
0131         else:
0132             ramcount = job_spec.minRamCount
0133
0134         # check min cores
0135         if self.mincore is not None and corecount < self.mincore:
0136             return False
0137
0138         # check max cores
0139         if self.maxcore is not None and corecount > self.maxcore:
0140             return False
0141
0142         # We assume ram unit is always MB
0143         ram_per_core = round(ramcount / corecount)
0144         # check min ram
0145         if self.minrampercore is not None and ram_per_core < self.minrampercore:
0146             return False
0147
0148         # check max ram
0149         if self.maxrampercore is not None and ram_per_core > self.maxrampercore:
0150             return False
0151
0152         return True
0153
0154     def is_single_core(self):
0155         if self.mincore is not None and self.mincore == 1 and self.maxcore is not None and self.maxcore == 1:
0156             return True
0157         return False
0158
0159     def is_multi_core(self):
0160         if self.mincore is not None and self.mincore > 1:
0161             return True
0162
0163     def column_names(cls, prefix=None):
0164         """
0165         return column names for DB interactions
0166         """
0167         ret = ""
0168         for attr in cls.attributes:
0169             if prefix is not None:
0170                 ret += f"{prefix}."
0171             ret += f"{attr},"
0172         ret = ret[:-1]
0173         return ret
0174
0175     column_names = classmethod(column_names)