CI/physmon/phys_perf_mon.sh

0001 #!/bin/bash
0002
0003 set -e
0004
0005 # helper function to selectively print and run commands without a subshell
0006 function run() {
0007     set -x
0008     "$@"
0009     # save exit code
0010     { rec=$?; } 2> /dev/null
0011     { set +x;   } 2> /dev/null
0012     # restore exit code
0013     (exit $rec)
0014 }
0015
0016 export run
0017
0018 run which python3
0019 shopt -s extglob
0020
0021
0022 mode=${1:-all}
0023 if ! [[ $mode = @(all|kf|gsf|gx2f|refit_kf|refit_gsf|fullchains|simulation) ]]; then
0024     echo "Usage: $0 <all|kf|gsf|gx2f|refit_kf|refit_gsf|fullchains|simulation> (outdir)"
0025     exit 1
0026 fi
0027
0028 outdir=${2:-physmon}
0029 mkdir -p $outdir
0030 mkdir -p $outdir/data
0031 mkdir -p $outdir/html
0032 mkdir -p $outdir/logs
0033
0034 refdir=CI/physmon/reference
0035 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}"  )" &> /dev/null && pwd  )
0036
0037 # File to accumulate the histcmp results
0038 histcmp_results=$outdir/histcmp_results.csv
0039 echo -n "" > $histcmp_results
0040
0041 memory_dir=${outdir}/memory
0042 mkdir -p "$memory_dir"
0043
0044 if [ "$(uname)" == "Darwin" ]; then
0045     function measure {
0046         label=$1
0047         slug=$2
0048         shift
0049         shift
0050         echo "Measure Darwin $label ($slug)" >&2
0051         tmp=$(mktemp)
0052         echo "+ $@" >&2
0053         /usr/bin/time -l -o "$tmp" "$@"
0054         # save exit code
0055         rec=$?
0056
0057         of="${memory_dir}/mem_${slug}.csv"
0058         {
0059             echo "# spyral-label: $label"
0060             echo "# spyral-cmd: $*"
0061             echo "time,rss,vms"
0062             # in bytes
0063             grep -E "real" "$tmp" | awk '{printf $1}'
0064             printf ","
0065             grep -E "maximum resident set size" "$tmp" | awk '{printf $1}'
0066             printf ",0\n"
0067         } > "$of"
0068         # restore exit code
0069         (exit $rec)
0070     }
0071     export measure
0072 elif [ "$(uname)" == "Linux" ]; then
0073     function measure {
0074         label=$1
0075         slug=$2
0076         shift
0077         shift
0078         echo "Measure Linux $label ($slug)" >&2
0079         tmp=$(mktemp)
0080         echo "+ $@" >&2
0081         /usr/bin/time -v -o  "$tmp" "$@"
0082         # save exit code
0083         rec=$?
0084         # in kbytes
0085         max_rss=$(grep "Maximum resident set size (kbytes):" "$tmp" | awk '{printf $(NF)}')
0086         max_rss=$(( 1024 * max_rss ))
0087         echo "Maximum resident set size: $(printf "%'d" $max_rss) bytes"
0088
0089         wall_time=$(grep "Elapsed (wall clock)" "$tmp" | awk '{printf $(NF)}')
0090         wall_time=$(python3 -c "i='${wall_time}';p=i.split(':');p = p if len(p) == 3 else ['0', *p];t=float(p[0])*60*60 + float(p[1])*60 + float(p[2]);print(t)")
0091         echo "Elapsed (wall clock) time: ${wall_time} seconds"
0092
0093         of="${memory_dir}/mem_${slug}.csv"
0094         {
0095             echo "# spyral-label: $label"
0096             echo "# spyral-cmd: $*"
0097             echo "time,rss,vms"
0098             echo "${wall_time},${max_rss},0"
0099         } > "$of"
0100         # restore exit code
0101         (exit $rec)
0102     }
0103     export measure
0104 else
0105     function measure {
0106         echo "Not measuring because unknown environment"
0107         shift
0108         shift
0109         "$@"
0110     }
0111     export measure
0112 fi
0113
0114 if [ -n "$CI" ]; then
0115     echo "CI mode, do not abort immediately on failure"
0116     set +e
0117 fi
0118 ec=0
0119
0120 source $SCRIPT_DIR/setup.sh
0121
0122 function run_physmon_gen() {
0123     title=$1
0124     slug=$2
0125
0126     script=CI/physmon/workflows/physmon_${slug}.py
0127
0128     mkdir -p $outdir/data/$slug
0129     mkdir -p $outdir/logs
0130     measure "$title" "$slug" ${script} $outdir/data/$slug 2>&1 > $outdir/logs/${slug}.log
0131
0132     this_ec=$?
0133     ec=$(($ec | $this_ec))
0134
0135     if [ $this_ec -ne 0 ]; then
0136         echo "::error::🟥 Dataset generation failed: ${script} -> ec=$this_ec"
0137     else
0138         echo "::notice::✅ Dataset generation succeeded: ${script}"
0139     fi
0140 }
0141
0142 echo "::group::Generate validation dataset"
0143 if [[ "$mode" == "all" || "$mode" == "simulation" ]]; then
0144     run_physmon_gen "Simulation" "simulation"
0145 fi
0146 if [[ "$mode" == "all" || "$mode" == "kf" ]]; then
0147     run_physmon_gen "Truth Tracking KF" "trackfitting_kf"
0148 fi
0149 if [[ "$mode" == "all" || "$mode" == "gsf" ]]; then
0150     run_physmon_gen "Truth Tracking GSF" "trackfitting_gsf"
0151 fi
0152 if [[ "$mode" == "all" || "$mode" == "gx2f" ]]; then
0153     run_physmon_gen "Truth Tracking GX2F" "trackfitting_gx2f"
0154 fi
0155 if [[ "$mode" == "all" || "$mode" == "refit_kf" ]]; then
0156     run_physmon_gen "Truth Tracking KF refit" "trackrefitting_kf"
0157 fi
0158 if [[ "$mode" == "all" || "$mode" == "refit_gsf" ]]; then
0159     run_physmon_gen "Truth Tracking GSF refit" "trackrefitting_gsf"
0160 fi
0161 if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
0162     run_physmon_gen "CKF single muon" "trackfinding_1muon"
0163     run_physmon_gen "CKF muon 50" "trackfinding_4muon_50vertices"
0164     run_physmon_gen "CKF ttbar 200" "trackfinding_ttbar_pu200"
0165 fi
0166 echo "::endgroup::"
0167
0168
0169 function run_histcmp() {
0170     a=$1
0171     b=$2
0172     title=$3
0173     html_path=$4
0174     plots_path=$5
0175     shift 5
0176
0177     echo "::group::Comparing $a vs. $b"
0178
0179     if [ ! -f "$a" ]; then
0180         echo "::error::histcmp failed: File $a does not exist"
0181         ec=1
0182     fi
0183
0184     if [ ! -f "$b" ]; then
0185         echo "::error::histcmp failed: File $b does not exist"
0186         ec=1
0187     fi
0188
0189     run histcmp $a $b \
0190         --label-reference=reference \
0191         --label-monitored=monitored \
0192         --title="$title" \
0193         -o $outdir/html/$html_path \
0194         -p $outdir/html/$plots_path \
0195         "$@"
0196
0197     this_ec=$?
0198     ec=$(($ec | $this_ec))
0199
0200     if [ $this_ec -ne 0 ]; then
0201         echo "::error::histcmp failed: ec=$this_ec"
0202     fi
0203
0204     echo "\"${title}\",html/${html_path},${this_ec}" >> $histcmp_results
0205
0206     echo "::endgroup::"
0207 }
0208
0209 function trackfinding() {
0210     name=$1
0211     path=$2
0212
0213     default_config="CI/physmon/config/default.yml"
0214
0215     if [ -f $refdir/$path/performance_seeding.root ]; then
0216         run_histcmp \
0217             $outdir/data/$path/performance_seeding.root \
0218             $refdir/$path/performance_seeding.root \
0219             "Seeding ${name}" \
0220             $path/performance_seeding.html \
0221             $path/performance_seeding_plots \
0222             --config $default_config
0223     fi
0224
0225     run_histcmp \
0226         $outdir/data/$path/performance_finding_ckf.root \
0227         $refdir/$path/performance_finding_ckf.root \
0228         "CKF finding performance | ${name}" \
0229         $path/performance_finding_ckf.html \
0230         $path/performance_finding_ckf_plots \
0231         --config $default_config
0232
0233     run_histcmp \
0234         $outdir/data/$path/performance_fitting_ckf.root \
0235         $refdir/$path/performance_fitting_ckf.root \
0236         "CKF fitting performance | ${name}" \
0237         $path/performance_fitting_ckf.html \
0238         $path/performance_fitting_ckf_plots \
0239         --config $default_config
0240
0241
0242     run Examples/Scripts/generic_plotter.py \
0243         $outdir/data/$path/tracksummary_ckf.root \
0244         tracksummary \
0245         $outdir/data/$path/tracksummary_ckf_hist.root \
0246         --silent \
0247         --config CI/physmon/config/tracksummary_ckf.yml
0248     ec=$(($ec | $?))
0249
0250     # remove ntuple file because it's large
0251     rm $outdir/data/$path/tracksummary_ckf.root
0252
0253     run_histcmp \
0254         $outdir/data/$path/tracksummary_ckf_hist.root \
0255         $refdir/$path/tracksummary_ckf_hist.root \
0256         "CKF track summary | ${name}" \
0257         $path/tracksummary_ckf.html \
0258         $path/tracksummary_ckf_plots
0259
0260     if [ -f $refdir/$path/performance_finding_ckf_ambi.root ]; then
0261         run_histcmp \
0262             $outdir/data/$path/performance_finding_ckf_ambi.root \
0263             $refdir/$path/performance_finding_ckf_ambi.root \
0264             "Ambisolver finding performance | ${name}" \
0265             $path/performance_finding_ckf_ambi.html \
0266             $path/performance_finding_ckf_ambi
0267     fi
0268
0269     if [ -f $refdir/$path/performance_finding_ckf_ml_solver.root ]; then
0270         run_histcmp \
0271             $outdir/data/$path/performance_finding_ckf_ml_solver.root \
0272             $refdir/$path/performance_finding_ckf_ml_solver.root \
0273             "ML Ambisolver | ${name}" \
0274             $path/performance_finding_ckf_ml_solver.html \
0275             $path/performance_finding_ckf_ml_solver
0276     fi
0277 }
0278
0279 function vertexing() {
0280     name=$1
0281     path=$2
0282     config=$3
0283
0284     if [ -f $refdir/$path/performance_vertexing_ivf_notime_hist.root ]; then
0285         run Examples/Scripts/generic_plotter.py \
0286             $outdir/data/$path/performance_vertexing_ivf_notime.root \
0287             vertexing \
0288             $outdir/data/$path/performance_vertexing_ivf_notime_hist.root \
0289             --silent \
0290             --config $config
0291         ec=$(($ec | $?))
0292
0293         # remove ntuple file because it's large
0294         rm $outdir/data/$path/performance_vertexing_ivf_notime.root
0295
0296         run_histcmp \
0297             $outdir/data/$path/performance_vertexing_ivf_notime_hist.root \
0298             $refdir/$path/performance_vertexing_ivf_notime_hist.root \
0299             "IVF notime | ${name}" \
0300             $path/performance_vertexing_ivf_notime.html \
0301             $path/performance_vertexing_ivf_notime_plots
0302     fi
0303
0304     run Examples/Scripts/generic_plotter.py \
0305         $outdir/data/$path/performance_vertexing_amvf_gauss_notime.root \
0306         vertexing \
0307         $outdir/data/$path/performance_vertexing_amvf_gauss_notime_hist.root \
0308         --silent \
0309         --config $config
0310     ec=$(($ec | $?))
0311
0312     # remove ntuple file because it's large
0313     rm $outdir/data/$path/performance_vertexing_amvf_gauss_notime.root
0314
0315     run_histcmp \
0316         $outdir/data/$path/performance_vertexing_amvf_gauss_notime_hist.root \
0317         $refdir/$path/performance_vertexing_amvf_gauss_notime_hist.root \
0318         "AMVF gauss notime | ${name}" \
0319         $path/performance_vertexing_amvf_gauss_notime.html \
0320         $path/performance_vertexing_amvf_gauss_notime_plots
0321
0322     run Examples/Scripts/generic_plotter.py \
0323         $outdir/data/$path/performance_vertexing_amvf_grid_time.root \
0324         vertexing \
0325         $outdir/data/$path/performance_vertexing_amvf_grid_time_hist.root \
0326         --silent \
0327         --config $config
0328     ec=$(($ec | $?))
0329
0330     # remove ntuple file because it's large
0331     rm $outdir/data/$path/performance_vertexing_amvf_grid_time.root
0332
0333     run_histcmp \
0334         $outdir/data/$path/performance_vertexing_amvf_grid_time_hist.root \
0335         $refdir/$path/performance_vertexing_amvf_grid_time_hist.root \
0336         "AMVF grid time | ${name}" \
0337         $path/performance_vertexing_amvf_grid_time.html \
0338         $path/performance_vertexing_amvf_grid_time_plots
0339 }
0340
0341 function simulation() {
0342     suffix=$1
0343
0344     config="CI/physmon/config/simulation.yml"
0345
0346     run Examples/Scripts/generic_plotter.py \
0347         $outdir/data/simulation/particles_${suffix}.root \
0348         particles \
0349         $outdir/data/simulation/particles_${suffix}_hist.root \
0350         --silent \
0351         --config $config
0352     ec=$(($ec | $?))
0353
0354     # remove ntuple file because it's large
0355     rm $outdir/data/simulation/particles_${suffix}.root
0356
0357     run_histcmp \
0358         $outdir/data/simulation/particles_${suffix}_hist.root \
0359         $refdir/simulation/particles_${suffix}_hist.root \
0360         "Particles ${suffix}" \
0361         simulation/particles_${suffix}.html \
0362         simulation/particles_${suffix}_plots
0363 }
0364
0365 function generation() {
0366     run Examples/Scripts/generic_plotter.py \
0367         $outdir/data/simulation/particles_ttbar.root \
0368         particles \
0369         $outdir/data/simulation/particles_ttbar_hist.root \
0370         --silent \
0371         --config CI/physmon/config/pythia8_ttbar.yml
0372
0373     # remove ntuple file because it's large
0374     rm $outdir/data/simulation/particles_ttbar.root
0375
0376     run_histcmp \
0377         $outdir/data/simulation/particles_ttbar_hist.root \
0378         $refdir/simulation/particles_ttbar_hist.root \
0379         "Particles ttbar" \
0380         simulation/particles_ttbar.html \
0381         simulation/particles_ttbar_plots
0382
0383     run Examples/Scripts/generic_plotter.py \
0384         $outdir/data/simulation/vertices_ttbar.root \
0385         vertices \
0386         $outdir/data/simulation/vertices_ttbar_hist.root \
0387         --silent \
0388         --config CI/physmon/config/pythia8_ttbar.yml
0389
0390     # remove ntuple file because it's large
0391     rm $outdir/data/simulation/vertices_ttbar.root
0392
0393     run_histcmp \
0394         $outdir/data/simulation/vertices_ttbar_hist.root \
0395         $refdir/simulation/vertices_ttbar_hist.root \
0396         "Vertices ttbar" \
0397         simulation/vertices_ttbar.html \
0398         simulation/vertices_ttbar_plots
0399 }
0400
0401 if [[ "$mode" == "all" || "$mode" == "simulation" ]]; then
0402     simulation fatras
0403     simulation geant4
0404
0405     generation
0406 fi
0407
0408 if [[ "$mode" == "all" || "$mode" == "kf" ]]; then
0409     run_histcmp \
0410         $outdir/data/trackfitting_kf/performance_trackfitting.root \
0411         $refdir/trackfitting_kf/performance_trackfitting.root \
0412         "Truth tracking (KF)" \
0413         trackfitting_kf/performance_trackfitting.html \
0414         trackfitting_kf/performance_trackfitting_plots \
0415         --config CI/physmon/config/trackfitting_kf.yml
0416 fi
0417
0418 if [[ "$mode" == "all" || "$mode" == "gsf" ]]; then
0419     run_histcmp \
0420         $outdir/data/trackfitting_gsf/performance_trackfitting.root \
0421         $refdir/trackfitting_gsf/performance_trackfitting.root \
0422         "Truth tracking (GSF)" \
0423         trackfitting_gsf/performance_trackfitting.html \
0424         trackfitting_gsf/performance_trackfitting_plots \
0425         --config CI/physmon/config/trackfitting_gsf.yml
0426 fi
0427
0428 if [[ "$mode" == "all" || "$mode" == "gx2f" ]]; then
0429     run_histcmp \
0430         $outdir/data/trackfitting_gx2f/performance_trackfitting.root \
0431         $refdir/trackfitting_gx2f/performance_trackfitting.root \
0432         "Truth tracking (GX2F)" \
0433         trackfitting_gx2f/performance_trackfitting.html \
0434         trackfitting_gx2f/performance_trackfitting_plots \
0435         --config CI/physmon/config/trackfitting_gx2f.yml
0436 fi
0437
0438 if [[ "$mode" == "all" || "$mode" == "kf_refit" ]]; then
0439     run_histcmp \
0440         $outdir/data/trackrefitting_kf/performance_trackrefitting.root \
0441         $refdir/trackrefitting_kf/performance_trackrefitting.root \
0442         "Truth tracking (KF refit)" \
0443         trackrefitting_kf/performance_trackrefitting.html \
0444         trackrefitting_kf/performance_trackrefitting_plots \
0445         --config CI/physmon/config/trackfitting_kf.yml
0446 fi
0447
0448 if [[ "$mode" == "all" || "$mode" == "gsf_refit" ]]; then
0449     run_histcmp \
0450         $outdir/data/trackrefitting_gsf/performance_trackrefitting.root \
0451         $refdir/trackrefitting_gsf/performance_trackrefitting.root \
0452         "Truth tracking (GSF refit)" \
0453         trackrefitting_gsf/performance_trackrefitting.html \
0454         trackrefitting_gsf/performance_trackrefitting_plots \
0455         --config CI/physmon/config/trackfitting_gsf.yml
0456 fi
0457
0458 if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
0459     trackfinding "trackfinding | single muon | truth smeared seeding" trackfinding_1muon/truth_smeared
0460     trackfinding "trackfinding | single muon | truth estimated seeding" trackfinding_1muon/truth_estimated
0461     trackfinding "trackfinding | single muon | default seeding" trackfinding_1muon/seeded
0462     trackfinding "trackfinding | single muon | orthogonal seeding" trackfinding_1muon/orthogonal
0463
0464     trackfinding "trackfinding | 4 muon x 50 vertices | default seeding" trackfinding_4muon_50vertices
0465     vertexing "trackfinding | 4 muon x 50 vertices | default seeding" trackfinding_4muon_50vertices CI/physmon/config/vertexing_4muon_50vertices.yml
0466
0467     trackfinding "trackfinding | ttbar with 200 pileup | default seeding" trackfinding_ttbar_pu200
0468     vertexing "trackfinding | ttbar with 200 pileup | default seeding" trackfinding_ttbar_pu200 CI/physmon/config/vertexing_ttbar_pu200.yml
0469 fi
0470
0471 run CI/physmon/summary.py $histcmp_results \
0472   --md $outdir/summary.md \
0473   --html $outdir/summary.html
0474 ec=$(($ec | $?))
0475
0476 exit $ec