Warning, /AID2E-framework/examples/epic/epic_imaging_optimization.yml is written in an unsupported language. File is not indexed.
0001 # ============================================================================
0002 # ePIC BIC Single-Objective Optimization
0003 # ============================================================================
0004 # This example shows how to configure and run an entire
0005 # ePIC workflow in YAML.
0006 #
0007 # Usage:
0008 # aid2e optimize epic_imaging_optimization.yml
0009 # ============================================================================
0010
0011 # SECTION 0: PROBLEM DEFINITION
0012 # What to optimize (objectives) and design space
0013 # ============================================================================
0014 problem:
0015 name: "BIC Single-Objective Optimization"
0016 problem_type: "EPIC_CALORIMETRY"
0017 description: "Single-objective optimization of the BIC for angular resolution"
0018 output_location: "epic_example_test"
0019 work_location: "epic_example_exec"
0020
0021 # ePIC Environment Configuration
0022 # ==============================
0023 # Configure paths to the ePIC software stack. Either eic_shell OR
0024 # singularity_image must be provided. If both are provided,
0025 # singularity_image takes precedence.
0026 epic_environment:
0027
0028 # Option A: Use eic-shell script
0029 eic_shell: "/path/to/my/eic-shell"
0030
0031 # Option B: Or use singularity image
0032 # singularity_image: "/path/to/my/eic_xl-nightly.sif"
0033
0034 # Path to ePIC installation directory. This
0035 # will be copied and modified during execution.
0036 epic_install: "./epic"
0037
0038 # Geometry configuration file to use (e.g. epic, epic_full)
0039 epic_config: "epic"
0040
0041 # ===========================================================================
0042 # PARAMETERS: Define what we're optimizing
0043 # ===========================================================================
0044 # Here, we want to find the optimal combination of AstroPix
0045 # staves in the BIC. Staves can be enabled (1) or disabled
0046 # (0) in their xml files.
0047 #
0048 # NOTE: the inline_design keyword indicates that parameters will
0049 # be defined directly in this file. Parameters can also be defined
0050 # in a separate file, such as
0051 #
0052 # examples/basic/design.params
0053 inline_design:
0054 epic_design_space:
0055 epic_design_parameters:
0056 bic:
0057 file_path: "compact/ecal/bic_default.xml"
0058 parameters:
0059
0060 enable_staves_2:
0061 value: 0
0062 choices: [0, 1]
0063 xml_path: ".//constant[@name='EcalBarrel_enable_staves_2']"
0064 attribute: "value"
0065 unit: ""
0066
0067 enable_staves_3:
0068 value: 1
0069 choices: [0, 1]
0070 xml_path: ".//constant[@name='EcalBarrel_enable_staves_3']"
0071 attribute: "value"
0072 unit: ""
0073
0074 enable_staves_4:
0075 value: 1
0076 choices: [0, 1]
0077 xml_path: ".//constant[@name='EcalBarrel_enable_staves_4']"
0078 attribute: "value"
0079 unit: ""
0080
0081 enable_staves_5:
0082 value: 0
0083 choices: [0, 1]
0084 xml_path: ".//constant[@name='EcalBarrel_enable_staves_5']"
0085 attribute: "value"
0086 unit: ""
0087
0088 enable_staves_6:
0089 value: 1
0090 choices: [0, 1]
0091 xml_path: ".//constant[@name='EcalBarrel_enable_staves_6']"
0092 attribute: "value"
0093 unit: ""
0094
0095 optimization_groups:
0096 default:
0097 - "bic.EcalBarrel_enable_staves_2"
0098 - "bic.EcalBarrel_enable_staves_3"
0099 - "bic.EcalBarrel_enable_staves_4"
0100 - "bic.EcalBarrel_enable_staves_5"
0101 - "bic.EcalBarrel_enable_staves_6"
0102
0103 # ========================================================================
0104 # OBJECTIVES: Define what we're optimizing for
0105 # ========================================================================
0106 # This a SINGLE objective: one run of the workflow
0107 # will output the calculated eta resolution, which
0108 # we want to minimize
0109 objectives:
0110 - name: "phi_resolution"
0111 direction: "minimize"
0112
0113 # Objective-level scheduler (applies to this objective's execution)
0114 scheduler:
0115 runner_type: "JobLibRunner"
0116 parameters:
0117 n_jobs: -1 # Use all available cores
0118 backend: "threading"
0119
0120 # Objective plan: HOW to compute eta_resolution
0121 # TODO this is the function to extract the objectives
0122 # -- should retrieve value from an output file (e.g. maybe a json file)
0123 # generated during workflow below
0124 # FIXME (DMA) the relationship between this and the workflows
0125 # isn't clear to me. We might consider renaming this to be
0126 # indicative of when to use plan vs. a workflow
0127 objective_plan:
0128 steps:
0129 stages:
0130 - name: "evaluate_phi_resolution"
0131 script:
0132 path: "scripts/test.py"
0133 output_file: "photon_phi_resolution.json"
0134 timeout_sec: 60 # should take no more than 1 min to extract
0135 produces_objective: true
0136
0137 # Metrics to extract from the objective plan output
0138 metrics_keys: ["phi_resolution"]
0139
0140 # SECTION 1: OPTIMIZER DEFINITION
0141 # How to optimize the defined problem
0142 # =============================================================================
0143 optimizer:
0144 name: "ax" # supported optimizers: "ax", "pymoo"
0145 type: "bayesian" # supported types: "bayesian" (Ax), "generative" (PyMoo)
0146
0147 # ===========================================================================
0148 # PARAMETERS: Opimitizer-specific strategy and options
0149 # ===========================================================================
0150 # Here, we define (1) our generation strategy -- how to propose new design
0151 # points -- and (2) all optimizer-specific options. In this case, we will
0152 # 1. Run n_initial_samples trials with design points generated quasi-
0153 # randomly (via a "Sobol sequence")
0154 # 2. And then run n_iterations - n_initial_samples trials with design
0155 # points generated by a "Modular BoTorch" model
0156 #
0157 # For more details on generation strategies in Ax and BoTorch, see
0158 #
0159 # https://ax.dev/docs/generation_strategy
0160 # https://botorch.org/docs/introduction
0161 #
0162 parameters:
0163 initialization_strategy: "sobol"
0164 generator: "BOTORCH_MODULAR"
0165
0166 # Options for Modular BoTorch generator
0167 generator_kwargs:
0168 botorch_acqf_class: "qLogNoisyExpectedHypervolumeImprovement"
0169 acquisition_options:
0170 prune_baseline: true
0171
0172 # Generarl generation strategy options
0173 generator_gen_kwargs:
0174 model_gen_options:
0175 optimizer_kwargs:
0176 sequential: false
0177 num_restarts: 10
0178
0179 # Minimum (or maximum) value of computed objectives to consider design
0180 # point as a viable candidate. Anything trial ABOVE (or BELOW if
0181 # maximizing) this will not be considered.
0182 objective_thresholds:
0183 phi_resolution: 1.0
0184
0185 n_initial_samples: 1 # number of initialization trials
0186 n_iterations: 2 # total number of trials to run
0187 batch_size: 1 # (optional) max number of trials to run in parallel
0188 seed: 42 # (optional) seed to use for random number generation
0189
0190 # SECTION 2: SCHEDULER CONFIGURATION
0191 # Global scheduler defaults (can be overridden at workflow/branch/stage level)
0192 # =============================================================================
0193 scheduler:
0194 runner_type: "JobLibRunner"
0195 parameters:
0196 n_jobs: 4 # Use 4 parallel jobs by default
0197 backend: "loky" # Use process-based parallelism (default)
0198
0199 # SECTION 3: WORKFLOW EXECUTION
0200 # How to execute the optimization (stages, parallelism, etc.)
0201 # =============================================================================
0202 # Here, we have only one workflow: optimize the number of Astropix layers
0203 # in the BIC. Multiple optimizations can be run in parallel, where each
0204 # one would have a corresponding workflow.
0205 #
0206 # Notice the 'stack_type' keyword: this indicates which software stack we're
0207 # going to use. For this example, we'll use the ePIC stack.
0208 #
0209 # NOTE: When objectives are simple (single script call), workflows are optional
0210 workflows:
0211 workflows:
0212 - name: "imaging_optimization"
0213 description: "Optimize the number of AstroPix layers in the BIC"
0214 stack_type: "epic"
0215
0216 # =========================================================================
0217 # BRANCHES: Parallel execution paths
0218 # =========================================================================
0219 # One optimization can have multiple branches which run in
0220 # parallel. These should correspond to independent pipelines.
0221 # For example, suppose we had 2 objectives,
0222 # 1. one for single particle simulations and
0223 # 2. another for an event generator.
0224 # These could be run in parallel as separate branches.
0225 branches:
0226 - name: "photon_phi_resolution"
0227 description: "Calculate phi resolution for single photons"
0228
0229 # =====================================================================
0230 # STAGES: Groups of sequential computation steps
0231 # =====================================================================
0232 # The workflow below has 3 stages:
0233 # 1. Run overlap checks on modified geometry
0234 # 2. Run Geant4 simulation via npsim for 3 kinematic points in parallel
0235 # 3. Merge npsim output, run eicrecon on merged output, and finally
0236 # run analysis on eicrecon output
0237 stages:
0238
0239 # Stage 1: run overlap checks
0240 # ==================================================================
0241 - name: "geo"
0242 description: "Check for overlaps in modified geomtry"
0243 jobs:
0244
0245 # JOBS: Individual steps in a stage of a workflow
0246 # ==============================================================
0247 # A stage can be composed of any number of jobs, which are
0248 # individual actions to be run. Details of how the job is
0249 # run are set in the "payload" block
0250 - name: "geo_job"
0251 payload:
0252 evaluator_type: "stack" # FIXME this should be default
0253 stack_type: "epic" # FIXME this should be default
0254 job_id: "geo"
0255
0256
0257 # LAYERS: Components of a software stack
0258 # ============================================================
0259 # A software stack is composed of one or more components,
0260 # such as npsim or EICrecon, referred to as a "layer" in
0261 # For a stack worklow, one or more layers can be run per job.
0262 #
0263 # The "name" keyowrd indicates which layer to run. For the
0264 # ePIC stack, we support:
0265 # -- geo: check for overlaps in geometry
0266 # -- sim: run npsim (Geant4)
0267 # -- rec: run EICrecon
0268 # -- ana: run user code or other commands (e.g. running
0269 # an analysis, compiling a plugin, etc.)
0270 layers:
0271 - name: "geo" # NOTE this should be a unique identifier for THIS instance of a layer
0272 layer: "geo"
0273 inputs:
0274 - "{{geometry_dir}}/install/share/epic/epic.xml"
0275 outputs:
0276 - "{{execution_dir}}/geo.overlaps.txt"
0277
0278 # Stage 2: run Geant4 simulations
0279 # =================================================================
0280 - name: "sim"
0281 description: "Simulate single photons"
0282 jobs:
0283
0284 # NOTE: anything in {{ }} will be substituted during execution
0285 - name: "sim_job"
0286 payload:
0287 evaluator_type: "stack" # FIXME this should be default
0288 stack_type: "epic" # FIXME this should be default
0289 job_id: "sim"
0290 layers:
0291 - name: "sim"
0292 layer: "sim"
0293 inputs:
0294 - "inputs/central_photons_bin{{job_id}}.py" # FIXME job ID might not be what we want here...
0295 outputs:
0296 - "{{execution_dir}}/central_photons_bin{{job_id}}.edm4hep.root"
0297
0298 # Now we're going to create 3 copies of the job with job IDs
0299 # 0, 1, 2. These correspond to the 3 kinematic points to run,
0300 # each of which is defined by a DD4hep steering file:
0301 # - central_electrons_bin0.py
0302 # - central_electrons_bin1.py
0303 # - central_electrons_bin2.py
0304 job_factory:
0305 type: "range"
0306 params:
0307 n: 3
0308
0309 # And here we specify how many copies we can run in parallel
0310 parallelism:
0311 max_concurrent: 3
0312 retry_max: 2
0313 timeout_sec: 3600 # should take no more than 30 min. per kinematic point to run
0314
0315 # Stage 3: Merge, run eicrecon, and do analysis
0316 # =================================================================
0317 - name: "merge_rec_ana"
0318 description: "Merge simulation output, run reconstruction and analysis"
0319 jobs:
0320 - name: "merge_rec_ana_job"
0321 payload:
0322 evaluator_type: "stack" # FIXME this should be default
0323 stack_type: "epic" # FIXME this should be default
0324 job_id: "merge_rec_ana"
0325
0326 # in this stage, we'll have 1 job where we run 3 layers:
0327 # 1. ana -- where we merge the 3 sim jobs
0328 # 2. rec -- where we run EICrecon on the merged sim output
0329 # 3. ana -- where we run an analysis script on the
0330 # EICrecon output
0331 layers:
0332
0333 # For 'ana' layers, you'll need to specify:
0334 # -- 'command', which sets which command (or executable)
0335 # to run; and
0336 # -- 'rule', which defines how arguments and the command
0337 # are put together on the command line
0338 # The keywords {{command}}, {{inputs}}, {{outputs}}, and
0339 # {{arguments}} will be appropriately subsituted during
0340 # execution.
0341 - name: "ana_merge"
0342 layer: "ana"
0343 inputs:
0344 - "{{outputs[sim:sim_job_0:sim](0)}}"
0345 - "{{outputs[sim:sim_job_1:sim](0)}}"
0346 - "{{outputs[sim:sim_job_2:sim](0)}}"
0347 outputs:
0348 - "{{execution_dir}}/central_photons.edm4hep.root"
0349 command: "hadd"
0350 rule: "{{command}} -f {{outputs}} {{inputs}}"
0351
0352 # For the other layers, 'command' and 'rule' are already
0353 # defined. The defaults can be overwritten if needed.
0354 - name: "rec"
0355 layer: "rec"
0356 inputs:
0357 - "{{outputs[merge_rec_ana:merge_rec_ana_job:ana_merge](0)}}"
0358 outputs:
0359 - "{{execution_dir}}/central_photons.edm4eic.root"
0360 arguments:
0361 - "-Pnthreads=8"
0362 - "-Peicrecon:LogLevel=debug"
0363
0364 - name: "ana_reso"
0365 layer: "ana"
0366 inputs:
0367 - "{{outputs[merge_rec_ana:merge_rec_ana_job:rec](0)}}"
0368 outputs:
0369 - "{{execution_dir}}/central_photons.hist.root"
0370 arguments:
0371 - "-c phi"
0372 - "-s 22"
0373 command: "scripts/bic_angular_reso.py"
0374 rule: "python {{command}} -i {{inputs}} -o {{outputs}} {{arguments}}"
