Warning, /AID2E-framework/examples/bic/workflow.yml is written in an unsupported language. File is not indexed.
0001 # ============================================================================
0002 # ePIC BIC Single-Objective Optimization
0003 # ============================================================================
0004 # This example shows how to configure and run an entire
0005 # ePIC workflow in YAML.
0006 #
0007 # Usage:
0008 # aid2e optimize epic_imaging_optimization.yml
0009 # ============================================================================
0010
0011 # SECTION 0: PROBLEM DEFINITION
0012 # What to optimize (objectives) and design space
0013 # ============================================================================
0014 problem:
0015 name: "BIC Single-Objective Optimization"
0016 problem_type: "EPIC_BIC"
0017 description: "Single-objective optimization of the BIC for angular resolution"
0018 output_location: "./output"
0019 work_location: "./work"
0020
0021 # ePIC Environment Configuration
0022 # ==============================
0023 # Configure paths to the ePIC software stack. Either eic_shell OR
0024 # singularity_image must be provided. If both are provided,
0025 # singularity_image takes precedence.
0026 epic_environment:
0027
0028 # Option A: Use eic-shell script
0029 eic_shell: "/path/to/my/eic-shell"
0030
0031 # Option B: Or use singularity image
0032 # singularity_image: "/path/to/my/eic_xl-nightly.sif"
0033
0034 # Path to ePIC installation directory. This
0035 # will be copied and modified during execution.
0036 epic_install: "./epic"
0037
0038 # Geometry configuration file to use (e.g. epic, epic_full)
0039 epic_config: "epic"
0040
0041 # Turn on/off recompilation of geometry after modifying it
0042 # in a trial. Options:
0043 # "build" -
0044 geometry_mode: "build"
0045
0046 # ===========================================================================
0047 # PARAMETERS: Define what we're optimizing
0048 # ===========================================================================
0049 # Here, we want to find the optimal combination of AstroPix
0050 # staves in the BIC. Staves can be enabled (1) or disabled
0051 # (0) in their xml files.
0052 #
0053 # NOTE: the inline_design keyword indicates that parameters will
0054 # be defined directly in this file. Parameters can also be defined
0055 # in a separate file, such as
0056 #
0057 # examples/basic/design.params
0058 inline_design:
0059 epic_design_space:
0060 epic_design_parameters:
0061 bic:
0062 #file_path: "$DETECTOR_PATH/compact/ecal/bic_default.xml"
0063 file_path: "compact/ecal/bic_default.xml"
0064 parameters:
0065
0066 enable_staves_2:
0067 value: 0
0068 choices: [0, 1]
0069 xml_path: ".//constant[@name='EcalBarrel_enable_staves_2']"
0070 attribute: "value"
0071 unit: ""
0072
0073 enable_staves_3:
0074 value: 1
0075 choices: [0, 1]
0076 xml_path: ".//constant[@name='EcalBarrel_enable_staves_3']"
0077 attribute: "value"
0078 unit: ""
0079
0080 enable_staves_4:
0081 value: 1
0082 choices: [0, 1]
0083 xml_path: ".//constant[@name='EcalBarrel_enable_staves_4']"
0084 attribute: "value"
0085 unit: ""
0086
0087 enable_staves_5:
0088 value: 0
0089 choices: [0, 1]
0090 xml_path: ".//constant[@name='EcalBarrel_enable_staves_5']"
0091 attribute: "value"
0092 unit: ""
0093
0094 enable_staves_6:
0095 value: 1
0096 choices: [0, 1]
0097 xml_path: ".//constant[@name='EcalBarrel_enable_staves_6']"
0098 attribute: "value"
0099 unit: ""
0100
0101 optimization_groups:
0102 default:
0103 - "bic.EcalBarrel_enable_staves_2"
0104 - "bic.EcalBarrel_enable_staves_3"
0105 - "bic.EcalBarrel_enable_staves_4"
0106 - "bic.EcalBarrel_enable_staves_5"
0107 - "bic.EcalBarrel_enable_staves_6"
0108
0109 # ========================================================================
0110 # OBJECTIVES: Define what we're optimizing for
0111 # ========================================================================
0112 # This a SINGLE objective: one run of the workflow
0113 # will output the calculated eta resolution, which
0114 # we want to minimize
0115 objectives:
0116 - name: "phi_resolution"
0117 direction: "minimize"
0118
0119 # SECTION 1: OPTIMIZER DEFINITION
0120 # How to optimize the defined problem
0121 # =============================================================================
0122 optimizer:
0123 name: "ax" # supported optimizers: "ax", "pymoo"
0124 type: "bayesian" # supported types: "bayesian" (Ax), "generative" (PyMoo)
0125
0126 # ===========================================================================
0127 # PARAMETERS: Opimitizer-specific strategy and options
0128 # ===========================================================================
0129 # Here, we define (1) our generation strategy -- how to propose new design
0130 # points -- and (2) all optimizer-specific options. In this case, we will
0131 # 1. Run n_initial_samples trials with design points generated quasi-
0132 # randomly (via a "Sobol sequence")
0133 # 2. And then run n_iterations - n_initial_samples trials with design
0134 # points generated by a "Modular BoTorch" model
0135 #
0136 # For more details on generation strategies in Ax and BoTorch, see
0137 #
0138 # https://ax.dev/docs/generation_strategy
0139 # https://botorch.org/docs/introduction
0140 #
0141 parameters:
0142 initialization_strategy: "sobol"
0143 generator: "BOTORCH_MODULAR"
0144
0145 # Options for Modular BoTorch generator
0146 generator_kwargs:
0147 botorch_acqf_class: "qLogNoisyExpectedImprovement"
0148 acquisition_options:
0149 prune_baseline: true
0150
0151 # General generation strategy options
0152 generator_gen_kwargs:
0153 model_gen_options:
0154 optimizer_kwargs:
0155 #sequential: false
0156 num_restarts: 10
0157
0158 # Minimum (or maximum) value of computed objectives to consider design
0159 # point as a viable candidate. Anything trial ABOVE (or BELOW if
0160 # maximizing) this will not be considered.
0161 objective_thresholds:
0162 phi_resolution: 1.0
0163
0164 n_initial_samples: 2 # number of initialization trials
0165 n_iterations: 5 # total number of trials to run
0166 batch_size: 1 # (optional) max number of trials to run in parallel
0167 seed: 999 # (optional) seed to use for random number generation
0168
0169 # SECTION 2: SCHEDULER CONFIGURATION
0170 # Global scheduler defaults (can be overridden at workflow/branch/stage level)
0171 # =============================================================================
0172 scheduler:
0173 runner_type: "JobLibRunner"
0174 parameters:
0175 n_jobs: 4 # Use 4 parallel jobs by default
0176 backend: "loky" # Use process-based parallelism (default)
0177
0178 # If you were to use Slurm, the configuration
0179 # would look like:
0180 #runner_type: "SlurmRunner"
0181 #parameters:
0182 # account: "<account>"
0183 # partition: "<partition>"
0184 # nodes: 1
0185 # ntasks: 1
0186 # cpus_per_task: 1
0187 # mem: 4G
0188 # time: "04:00:00"
0189 # poll_interval: 10
0190 # capture_stdout: true
0191 # capture_stderr: true
0192 # runtime_working_dir: "/path/to/this/directory"
0193
0194 # SECTION 3: WORKFLOW EXECUTION
0195 # How to execute the optimization (stages, parallelism, etc.)
0196 # =============================================================================
0197 # Here, we have only one workflow: optimize the number of Astropix layers
0198 # in the BIC. Multiple optimizations can be run in parallel, where each
0199 # one would have a corresponding workflow.
0200 #
0201 # Notice the 'stack_type' keyword: this indicates which software stack we're
0202 # going to use. For this example, we'll use the ePIC stack.
0203 #
0204 # NOTE: When objectives are simple (single script call), workflows are optional
0205 workflows:
0206 workflows:
0207 - name: "imaging_optimization"
0208 description: "Optimize the number of AstroPix layers in the BIC"
0209 stack_type: "epic"
0210
0211 # =========================================================================
0212 # BRANCHES: Parallel execution paths
0213 # =========================================================================
0214 # One optimization can have multiple branches which run in
0215 # parallel. These should correspond to independent pipelines.
0216 # For example, suppose we had 2 objectives,
0217 # 1. one for single particle simulations and
0218 # 2. another for an event generator.
0219 # These could be run in parallel as separate branches.
0220 branches:
0221 - name: "photon_phi_resolution"
0222 description: "Calculate phi resolution for single photons"
0223
0224 # =====================================================================
0225 # STAGES: Groups of sequential computation steps
0226 # =====================================================================
0227 # The workflow below has 3 stages:
0228 # 1. Run overlap checks on modified geometry
0229 # 2. Run Geant4 simulation via npsim for 3 kinematic points in parallel
0230 # 3. Merge npsim output, run eicrecon on merged output, and finally
0231 # run analysis on eicrecon output
0232 stages:
0233
0234 # Stage 1: run overlap checks
0235 # ==================================================================
0236 - name: "geo"
0237 description: "Check for overlaps in modified geomtry"
0238 jobs:
0239
0240 # JOBS: Individual steps in a stage of a workflow
0241 # ==============================================================
0242 # A stage can be composed of any number of jobs, which are
0243 # individual actions to be run. Details of how the job is
0244 # run are set in the "payload" block
0245 - name: "geo_job"
0246 payload:
0247 evaluator_type: "stack" # FIXME this should be default
0248 stack_type: "epic" # FIXME this should be default
0249 job_id: "geo"
0250
0251
0252 # LAYERS: Components of a software stack
0253 # ============================================================
0254 # A software stack is composed of one or more components,
0255 # such as npsim or EICrecon, referred to as a "layer" in
0256 # For a stack worklow, one or more layers can be run per job.
0257 #
0258 # The "name" keyowrd indicates which layer to run. For the
0259 # ePIC stack, we support:
0260 # -- geo: check for overlaps in geometry
0261 # -- sim: run npsim (Geant4)
0262 # -- rec: run EICrecon
0263 # -- ana: run user code or other commands (e.g. running
0264 # an analysis, compiling a plugin, etc.)
0265 layers:
0266 - name: "geo" # NOTE this should be a unique identifier for THIS instance of a layer
0267 layer: "geo"
0268 inputs:
0269 - "{{geometry_dir}}/install/share/epic/epic.xml"
0270 outputs:
0271 - "{{output_dir}}/geo.overlaps.txt"
0272
0273 # Stage 2: run Geant4 simulations
0274 # =================================================================
0275 - name: "sim"
0276 description: "Simulate single photons"
0277 jobs:
0278
0279 # NOTE: anything in {{ }} will be substituted during execution
0280 - name: "sim_job"
0281 payload:
0282 evaluator_type: "stack" # FIXME this should be default
0283 stack_type: "epic" # FIXME this should be default
0284 job_id: "sim"
0285 layers:
0286 - name: "sim"
0287 layer: "sim"
0288 inputs:
0289 - "inputs/central_photons_bin{{job_index}}.py"
0290 outputs:
0291 - "{{output_dir}}/central_photons_bin{{job_index}}.edm4hep.root"
0292
0293 # Now we're going to create 3 copies of the job with job IDs
0294 # 0, 1, 2. These correspond to the 3 kinematic points to run,
0295 # each of which is defined by a DD4hep steering file:
0296 # - central_electrons_bin0.py
0297 # - central_electrons_bin1.py
0298 # - central_electrons_bin2.py
0299 job_factory:
0300 type: "range"
0301 params:
0302 n: 3
0303
0304 # And here we specify how many copies we can run in parallel
0305 parallelism:
0306 max_concurrent: 3
0307 retry_max: 2
0308 timeout_sec: 3600 # should take no more than 30 min. per kinematic point to run
0309
0310 # Stage 3: Merge, run eicrecon, and do analysis
0311 # =================================================================
0312 - name: "merge_rec_ana"
0313 description: "Merge simulation output, run reconstruction and analysis"
0314 jobs:
0315 - name: "merge_rec_ana_job"
0316 payload:
0317 evaluator_type: "stack" # FIXME this should be default
0318 stack_type: "epic" # FIXME this should be default
0319 job_id: "merge_rec_ana"
0320
0321 # in this stage, we'll have 1 job where we run 3 layers:
0322 # 1. ana -- where we merge the 3 sim jobs
0323 # 2. rec -- where we run EICrecon on the merged sim output
0324 # 3. ana -- where we run an analysis script on the
0325 # EICrecon output
0326 layers:
0327
0328 # For 'ana' layers, you'll need to specify:
0329 # -- 'command', which sets which command (or executable)
0330 # to run; and
0331 # -- 'rule', which defines how arguments and the command
0332 # are put together on the command line
0333 # The keywords {{command}}, {{inputs}}, {{outputs}}, and
0334 # {{arguments}} will be appropriately subsituted during
0335 # execution.
0336 - name: "ana_merge"
0337 layer: "ana"
0338 inputs:
0339 - "{{outputs[sim:sim_job_0:sim](0)}}"
0340 - "{{outputs[sim:sim_job_1:sim](0)}}"
0341 - "{{outputs[sim:sim_job_2:sim](0)}}"
0342 outputs:
0343 - "{{output_dir}}/central_photons.edm4hep.root"
0344 command: "hadd"
0345 rule: "{{command}} -f {{outputs}} {{inputs}}"
0346
0347 # For the other layers, 'command' and 'rule' are already
0348 # defined. The defaults can be overwritten if needed.
0349 - name: "rec"
0350 layer: "rec"
0351 inputs:
0352 - "{{outputs[merge_rec_ana:merge_rec_ana_job:ana_merge](0)}}"
0353 outputs:
0354 - "{{output_dir}}/central_photons.edm4eic.root"
0355 arguments:
0356 - "-Pnthreads=8"
0357 - "-Peicrecon:LogLevel=debug"
0358
0359 - name: "ana_reso"
0360 layer: "ana"
0361 inputs:
0362 - "{{outputs[merge_rec_ana:merge_rec_ana_job:rec](0)}}"
0363 outputs:
0364 - "{{output_dir}}/central_photons.hist.root"
0365 arguments:
0366 - "-c phi"
0367 - "-s 22"
0368 command: "{{workflow[config_dir]}}/scripts/bic_angular_reso.py"
0369 rule: "python {{command}} -i {{inputs}} -o {{outputs}} {{arguments}}"
0370
0371 # Stage 4: Extract objective from output of merge_rec_ana stage
0372 # ==================================================================
0373 - name: "extract_objectives"
0374 description: "Extract objectives from output"
0375 objective_plan:
0376 steps:
0377 stages:
0378 - name: "extract"
0379 inline:
0380 entrypoint: "scripts.extract_objective:ExtractObjective"
0381 extra_args:
0382 key: "phi_resolution"
0383 file: "{{outputs[merge_rec_ana:merge_rec_ana_job:ana_reso](0)}}" # FIXME use artifacts instead
0384 produces_objective: true
