Scripts/Optimization/Orion_tuning.py

0001 #!/usr/bin/env python3
0002 import os
0003 import matplotlib
0004 import subprocess
0005 from pathlib import Path
0006
0007 from orion.client import build_experiment
0008
0009 from Optuna_tuning import get_tracking_perf, run_ckf
0010
0011 matplotlib.use("pdf")
0012
0013 srcDir = Path(__file__).resolve().parent
0014
0015
0016 def run_ckf(params, names, outDir):
0017     if len(params) != len(names):
0018         raise Exception("Length of Params must equal names")
0019
0020     ckf_script = srcDir / "ckf.py"
0021     nevts = "--nEvents=1"
0022     indir = "--indir=" + str(srcDir)
0023     outdir = "--output=" + str(outDir)
0024
0025     ret = ["python"]
0026     ret.append(ckf_script)
0027     ret.append(nevts)
0028     ret.append(indir)
0029     ret.append(outdir)
0030
0031     i = 0
0032     for param in params:
0033         arg = "--sf_" + names[i] + "=" + str(param)
0034         ret.append(arg)
0035         i += 1
0036
0037     # Run CKF for the given parameters
0038     subprocess.call(ret)
0039
0040
0041 class Objective:
0042     def __init__(self, k_dup, k_time):
0043         self.res = {
0044             "eff": [],
0045             "fakerate": [],
0046             "duplicaterate": [],
0047             "runtime": [],
0048         }
0049
0050         self.k_dup = k_dup
0051         self.k_time = k_time
0052
0053     def __call__(
0054         self,
0055         maxSeedsPerSpM,
0056         cotThetaMax,
0057         sigmaScattering,
0058         radLengthPerSeed,
0059         impactMax,
0060         maxPtScattering,
0061         deltaRMin,
0062         deltaRMax,
0063         ckf_perf=True,
0064     ):
0065         params = [
0066             maxSeedsPerSpM,
0067             cotThetaMax,
0068             sigmaScattering,
0069             radLengthPerSeed,
0070             impactMax,
0071             maxPtScattering,
0072             deltaRMin,
0073             deltaRMax,
0074         ]
0075         keys = [
0076             "maxSeedsPerSpM",
0077             "cotThetaMax",
0078             "sigmaScattering",
0079             "radLengthPerSeed",
0080             "impactMax",
0081             "maxPtScattering",
0082             "deltaRMin",
0083             "deltaRMax",
0084         ]
0085
0086         get_tracking_perf(self, ckf_perf, params, keys)
0087
0088         efficiency = self.res["eff"][-1]
0089         penalty = (
0090             self.res["fakerate"][-1]
0091             + self.res["duplicaterate"][-1] / self.k_dup
0092             + self.res["runtime"][-1] / self.k_time
0093         )
0094
0095         return [
0096             {"name": "objective", "type": "objective", "value": -(efficiency - penalty)}
0097         ]
0098
0099
0100 def main():
0101     k_dup = 5
0102     k_time = 5
0103
0104     # Initializing the objective (score) function
0105     objective = Objective(k_dup, k_time)
0106
0107     # Defining the parameter space
0108     space = {
0109         "maxSeedsPerSpM": "uniform(0,10,discrete=True)",
0110         "cotThetaMax": "uniform(5.0,10.0)",
0111         "sigmaScattering": "uniform(0.2,50.0)",
0112         "radLengthPerSeed": "uniform(.001,0.1)",
0113         "impactMax": "uniform(0.1,25.0)",
0114         "maxPtScattering": "uniform(1.0, 50.0)",
0115         "deltaRMin": "uniform(0.25, 30.0)",
0116         "deltaRMax": "uniform(50.0,300.0)",
0117     }
0118
0119     # Remove storage file if already exists (conflicts with present run if not deleted)
0120     if os.path.exists("./db.pkl"):
0121         os.remove("./db.pkl")
0122
0123     # location to store metadata
0124     storage = {
0125         "type": "legacy",
0126         "database": {
0127             "type": "pickleddb",
0128             "host": "./db.pkl",
0129         },
0130     }
0131
0132     # Build new orion experiment
0133     experiment = build_experiment(
0134         "orion_new",
0135         space=space,
0136         storage=storage,
0137     )
0138
0139     # Start Optimization
0140     experiment.workon(objective, max_trials=3)
0141
0142     outputDir = Path("OrionResults")
0143     outputDir.mkdir(exist_ok=True)
0144
0145     # fetching trials in a dataframe
0146     df = experiment.to_pandas()
0147     df.to_csv(outputDir / "results.txt")
0148
0149     # Getting the best parameters
0150     df_imp = df[
0151         [
0152             "objective",
0153             "maxSeedsPerSpM",
0154             "cotThetaMax",
0155             "sigmaScattering",
0156             "radLengthPerSeed",
0157             "impactMax",
0158             "maxPtScattering",
0159             "deltaRMin",
0160             "deltaRMax",
0161         ]
0162     ]
0163     df_obj = df["objective"]
0164     min_obj = df_obj.min()
0165     df_final = df_imp[df_imp["objective"] == min_obj]
0166     print("Best Score = %s" % (df_final["objective"]))
0167     print("maxSeedsPerSpM = %s" % (df_final["maxSeedsPerSpM"]))
0168     print("cotThetaMax = %s" % (df_final["cotThetaMax"]))
0169     print("sigmaScattering = %s" % (df_final["sigmaScattering"]))
0170     print("radLengthPerSeed = %s" % (df_final["radLengthPerSeed"]))
0171     print("impactMax = %s" % (df_final["impactMax"]))
0172     print("maxPtScattering = %s" % (df_final["maxPtScattering"]))
0173     print("deltaRMin = %s" % (df_final["deltaRMin"]))
0174     print("deltaRMax = %s" % (df_final["deltaRMax"]))
0175
0176
0177 if __name__ == "__main__":
0178     main()