CI/physmon/phys_perf_mon.sh

0001 #!/bin/bash
0002
0003 set -e
0004
0005 # helper function to selectively print and run commands without a subshell
0006 function run() {
0007     set -x
0008     "$@"
0009     # save exit code
0010     { rec=$?; } 2> /dev/null
0011     { set +x;   } 2> /dev/null
0012     # restore exit code
0013     (exit $rec)
0014 }
0015
0016 export run
0017
0018 shopt -s extglob
0019
0020
0021 mode=${1:-all}
0022 if ! [[ $mode = @(all|kf|gsf|gx2f|refit_kf|refit_gsf|fullchains|simulation|gx2f_vs_kf) ]]; then
0023     echo "Usage: $0 <all|kf|gsf|gx2f|refit_kf|refit_gsf|fullchains|simulation|gx2f_vs_kf> (outdir)"
0024     exit 1
0025 fi
0026
0027 outdir=${2:-physmon}
0028 mkdir -p $outdir
0029 mkdir -p $outdir/data
0030 mkdir -p $outdir/html
0031 mkdir -p $outdir/logs
0032
0033 refdir=CI/physmon/reference
0034 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}"  )" &> /dev/null && pwd  )
0035
0036 # File to accumulate the histcmp results
0037 histcmp_results=$outdir/histcmp_results.csv
0038 echo -n "" > $histcmp_results
0039
0040 memory_dir=${outdir}/memory
0041 mkdir -p "$memory_dir"
0042
0043 if [ "$(uname)" == "Darwin" ]; then
0044     function measure {
0045         label=$1
0046         slug=$2
0047         shift
0048         shift
0049         echo "Measure Darwin $label ($slug)" >&2
0050         tmp=$(mktemp)
0051         echo "+ $@" >&2
0052         /usr/bin/time -l -o "$tmp" "$@"
0053         # save exit code
0054         rec=$?
0055
0056         of="${memory_dir}/mem_${slug}.csv"
0057         {
0058             echo "# spyral-label: $label"
0059             echo "# spyral-cmd: $*"
0060             echo "time,rss,vms"
0061             # in bytes
0062             grep -E "real" "$tmp" | awk '{printf $1}'
0063             printf ","
0064             grep -E "maximum resident set size" "$tmp" | awk '{printf $1}'
0065             printf ",0\n"
0066         } > "$of"
0067         # restore exit code
0068         (exit $rec)
0069     }
0070     export measure
0071 elif [ "$(uname)" == "Linux" ]; then
0072     function measure {
0073         label=$1
0074         slug=$2
0075         shift
0076         shift
0077         echo "Measure Linux $label ($slug)" >&2
0078         tmp=$(mktemp)
0079         echo "+ $@" >&2
0080         /usr/bin/time -v -o  "$tmp" "$@"
0081         # save exit code
0082         rec=$?
0083         # in kbytes
0084         max_rss=$(grep "Maximum resident set size (kbytes):" "$tmp" | awk '{printf $(NF)}')
0085         max_rss=$(( 1024 * max_rss ))
0086         echo "Maximum resident set size: $(printf "%'d" $max_rss) bytes"
0087
0088         wall_time=$(grep "Elapsed (wall clock)" "$tmp" | awk '{printf $(NF)}')
0089         wall_time=$(python3 -c "i='${wall_time}';p=i.split(':');p = p if len(p) == 3 else ['0', *p];t=float(p[0])*60*60 + float(p[1])*60 + float(p[2]);print(t)")
0090         echo "Elapsed (wall clock) time: ${wall_time} seconds"
0091
0092         of="${memory_dir}/mem_${slug}.csv"
0093         {
0094             echo "# spyral-label: $label"
0095             echo "# spyral-cmd: $*"
0096             echo "time,rss,vms"
0097             echo "${wall_time},${max_rss},0"
0098         } > "$of"
0099         # restore exit code
0100         (exit $rec)
0101     }
0102     export measure
0103 else
0104     function measure {
0105         echo "Not measuring because unknown environment"
0106         shift
0107         shift
0108         "$@"
0109     }
0110     export measure
0111 fi
0112
0113 if [ -n "$CI" ]; then
0114     echo "CI mode, do not abort immediately on failure"
0115     set +e
0116 fi
0117 ec=0
0118
0119 source $SCRIPT_DIR/setup.sh
0120
0121 function run_physmon_gen() {
0122     title=$1
0123     slug=$2
0124
0125     script=CI/physmon/workflows/physmon_${slug}.py
0126
0127     mkdir -p $outdir/data/$slug
0128     mkdir -p $outdir/logs
0129     measure "$title" "$slug" python3 ${script} $outdir/data/$slug 2>&1 > $outdir/logs/${slug}.log
0130
0131     this_ec=$?
0132     ec=$(($ec | $this_ec))
0133
0134     if [ $this_ec -ne 0 ]; then
0135         echo "::error::🟥 Dataset generation failed: ${script} -> ec=$this_ec"
0136     else
0137         echo "::notice::✅ Dataset generation succeeded: ${script}"
0138     fi
0139 }
0140
0141 echo "::group::Generate validation dataset"
0142 if [[ "$mode" == "all" || "$mode" == "simulation" ]]; then
0143     run_physmon_gen "Simulation" "simulation"
0144 fi
0145 if [[ "$mode" == "all" || "$mode" == "gsf" || "$mode" == "refit_gsf" ]]; then
0146     run_physmon_gen "Geant4 Sim for GSF" "simulation_gsf"
0147 fi
0148 if [[ "$mode" == "all" || "$mode" == "kf" ]]; then
0149     run_physmon_gen "Truth Tracking KF" "trackfitting_kf"
0150 fi
0151 if [[ "$mode" == "all" || "$mode" == "gsf" ]]; then
0152     run_physmon_gen "Truth Tracking GSF" "trackfitting_gsf"
0153 fi
0154 if [[ "$mode" == "all" || "$mode" == "gx2f" ]]; then
0155     run_physmon_gen "Truth Tracking GX2F" "trackfitting_gx2f"
0156 fi
0157 if [[ "$mode" == "all" || "$mode" == "refit_kf" ]]; then
0158     run_physmon_gen "Truth Tracking KF refit" "trackrefitting_kf"
0159 fi
0160 if [[ "$mode" == "all" || "$mode" == "refit_gsf" ]]; then
0161     run_physmon_gen "Truth Tracking GSF refit" "trackrefitting_gsf"
0162 fi
0163 if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
0164     run_physmon_gen "CKF single muon" "trackfinding_1muon"
0165     run_physmon_gen "CKF muon 50" "trackfinding_4muon_50vertices"
0166     run_physmon_gen "CKF ttbar 200" "trackfinding_ttbar_pu200"
0167 fi
0168 if [[ "$mode" == "all" || "$mode" == "gx2f_vs_kf" ]]; then
0169     run_physmon_gen "Comparison - Truth Tracking GX2F vs KF" "trackfitting_gx2f_vs_kf"
0170 fi
0171 echo "::endgroup::"
0172
0173
0174 function run_histcmp() {
0175     a=$1
0176     b=$2
0177     title=$3
0178     html_path=$4
0179     plots_path=$5
0180     shift 5
0181
0182     echo "::group::Comparing $a vs. $b"
0183
0184     if [ ! -f "$a" ]; then
0185         echo "::error::histcmp failed: File $a does not exist"
0186         ec=1
0187     fi
0188
0189     if [ ! -f "$b" ]; then
0190         echo "::error::histcmp failed: File $b does not exist"
0191         ec=1
0192     fi
0193
0194     run histcmp $a $b \
0195         --label-reference=reference \
0196         --label-monitored=monitored \
0197         --title="$title" \
0198         -o $outdir/html/$html_path \
0199         -p $outdir/html/$plots_path \
0200         "$@"
0201
0202     this_ec=$?
0203     ec=$(($ec | $this_ec))
0204
0205     if [ $this_ec -ne 0 ]; then
0206         echo "::error::histcmp failed: ec=$this_ec"
0207     fi
0208
0209     echo "\"${title}\",html/${html_path},${this_ec}" >> $histcmp_results
0210
0211     echo "::endgroup::"
0212 }
0213
0214 function trackfinding() {
0215     name=$1
0216     path=$2
0217
0218     default_config="CI/physmon/config/default.yml"
0219
0220     if [ -f $refdir/$path/performance_seeding.root ]; then
0221         run_histcmp \
0222             $outdir/data/$path/performance_seeding.root \
0223             $refdir/$path/performance_seeding.root \
0224             "Seeding ${name}" \
0225             $path/performance_seeding.html \
0226             $path/performance_seeding_plots \
0227             --config $default_config
0228     fi
0229
0230     run_histcmp \
0231         $outdir/data/$path/performance_finding_ckf.root \
0232         $refdir/$path/performance_finding_ckf.root \
0233         "CKF finding performance | ${name}" \
0234         $path/performance_finding_ckf.html \
0235         $path/performance_finding_ckf_plots \
0236         --config $default_config
0237
0238     run_histcmp \
0239         $outdir/data/$path/performance_fitting_ckf.root \
0240         $refdir/$path/performance_fitting_ckf.root \
0241         "CKF fitting performance | ${name}" \
0242         $path/performance_fitting_ckf.html \
0243         $path/performance_fitting_ckf_plots \
0244         --config $default_config
0245
0246     # TODO remove
0247     echo "which python3: $(which python3)"
0248     echo "python3 version: $(python3 --version)"
0249     echo "python3 -m pip list: $(python3 -m pip list)"
0250     run python3 Examples/Scripts/generic_plotter.py \
0251         $outdir/data/$path/tracksummary_ckf.root \
0252         tracksummary \
0253         $outdir/data/$path/tracksummary_ckf_hist.root \
0254         --silent \
0255         --config CI/physmon/config/tracksummary_ckf.yml
0256     ec=$(($ec | $?))
0257
0258     # remove ntuple file because it's large
0259     rm $outdir/data/$path/tracksummary_ckf.root
0260
0261     run_histcmp \
0262         $outdir/data/$path/tracksummary_ckf_hist.root \
0263         $refdir/$path/tracksummary_ckf_hist.root \
0264         "CKF track summary | ${name}" \
0265         $path/tracksummary_ckf.html \
0266         $path/tracksummary_ckf_plots
0267
0268     if [ -f $refdir/$path/performance_finding_ckf_ambi.root ]; then
0269         run_histcmp \
0270             $outdir/data/$path/performance_finding_ckf_ambi.root \
0271             $refdir/$path/performance_finding_ckf_ambi.root \
0272             "Ambisolver finding performance | ${name}" \
0273             $path/performance_finding_ckf_ambi.html \
0274             $path/performance_finding_ckf_ambi
0275     fi
0276
0277     if [ -f $refdir/$path/performance_finding_ckf_ml_solver.root ]; then
0278         run_histcmp \
0279             $outdir/data/$path/performance_finding_ckf_ml_solver.root \
0280             $refdir/$path/performance_finding_ckf_ml_solver.root \
0281             "ML Ambisolver | ${name}" \
0282             $path/performance_finding_ckf_ml_solver.html \
0283             $path/performance_finding_ckf_ml_solver
0284     fi
0285 }
0286
0287 function vertexing() {
0288     name=$1
0289     path=$2
0290     config=$3
0291
0292     if [ -f $refdir/$path/performance_vertexing_ivf_notime_hist.root ]; then
0293         run python3 Examples/Scripts/generic_plotter.py \
0294             $outdir/data/$path/performance_vertexing_ivf_notime.root \
0295             vertexing \
0296             $outdir/data/$path/performance_vertexing_ivf_notime_hist.root \
0297             --silent \
0298             --config $config
0299         ec=$(($ec | $?))
0300
0301         # remove ntuple file because it's large
0302         rm $outdir/data/$path/performance_vertexing_ivf_notime.root
0303
0304         run_histcmp \
0305             $outdir/data/$path/performance_vertexing_ivf_notime_hist.root \
0306             $refdir/$path/performance_vertexing_ivf_notime_hist.root \
0307             "IVF notime | ${name}" \
0308             $path/performance_vertexing_ivf_notime.html \
0309             $path/performance_vertexing_ivf_notime_plots
0310     fi
0311
0312     run python3 Examples/Scripts/generic_plotter.py \
0313         $outdir/data/$path/performance_vertexing_amvf_gauss_notime.root \
0314         vertexing \
0315         $outdir/data/$path/performance_vertexing_amvf_gauss_notime_hist.root \
0316         --silent \
0317         --config $config
0318     ec=$(($ec | $?))
0319
0320     # remove ntuple file because it's large
0321     rm $outdir/data/$path/performance_vertexing_amvf_gauss_notime.root
0322
0323     run_histcmp \
0324         $outdir/data/$path/performance_vertexing_amvf_gauss_notime_hist.root \
0325         $refdir/$path/performance_vertexing_amvf_gauss_notime_hist.root \
0326         "AMVF gauss notime | ${name}" \
0327         $path/performance_vertexing_amvf_gauss_notime.html \
0328         $path/performance_vertexing_amvf_gauss_notime_plots
0329
0330     run python3 Examples/Scripts/generic_plotter.py \
0331         $outdir/data/$path/performance_vertexing_amvf_grid_time.root \
0332         vertexing \
0333         $outdir/data/$path/performance_vertexing_amvf_grid_time_hist.root \
0334         --silent \
0335         --config $config
0336     ec=$(($ec | $?))
0337
0338     # remove ntuple file because it's large
0339     rm $outdir/data/$path/performance_vertexing_amvf_grid_time.root
0340
0341     run_histcmp \
0342         $outdir/data/$path/performance_vertexing_amvf_grid_time_hist.root \
0343         $refdir/$path/performance_vertexing_amvf_grid_time_hist.root \
0344         "AMVF grid time | ${name}" \
0345         $path/performance_vertexing_amvf_grid_time.html \
0346         $path/performance_vertexing_amvf_grid_time_plots
0347 }
0348
0349 function simulation() {
0350     suffix=$1
0351
0352     config="CI/physmon/config/simulation.yml"
0353
0354     run python3 Examples/Scripts/generic_plotter.py \
0355         $outdir/data/simulation/particles_${suffix}.root \
0356         particles \
0357         $outdir/data/simulation/particles_${suffix}_hist.root \
0358         --silent \
0359         --config $config
0360     ec=$(($ec | $?))
0361
0362     # remove ntuple file because it's large
0363     rm $outdir/data/simulation/particles_${suffix}.root
0364
0365     run_histcmp \
0366         $outdir/data/simulation/particles_${suffix}_hist.root \
0367         $refdir/simulation/particles_${suffix}_hist.root \
0368         "Particles ${suffix}" \
0369         simulation/particles_${suffix}.html \
0370         simulation/particles_${suffix}_plots
0371 }
0372
0373 function generation() {
0374     run python3 Examples/Scripts/generic_plotter.py \
0375         $outdir/data/simulation/particles_ttbar.root \
0376         particles \
0377         $outdir/data/simulation/particles_ttbar_hist.root \
0378         --silent \
0379         --config CI/physmon/config/pythia8_ttbar.yml
0380
0381     # remove ntuple file because it's large
0382     rm $outdir/data/simulation/particles_ttbar.root
0383
0384     run_histcmp \
0385         $outdir/data/simulation/particles_ttbar_hist.root \
0386         $refdir/simulation/particles_ttbar_hist.root \
0387         "Particles ttbar" \
0388         simulation/particles_ttbar.html \
0389         simulation/particles_ttbar_plots
0390
0391     run python3 Examples/Scripts/generic_plotter.py \
0392         $outdir/data/simulation/vertices_ttbar.root \
0393         vertices \
0394         $outdir/data/simulation/vertices_ttbar_hist.root \
0395         --silent \
0396         --config CI/physmon/config/pythia8_ttbar.yml
0397
0398     # remove ntuple file because it's large
0399     rm $outdir/data/simulation/vertices_ttbar.root
0400
0401     run_histcmp \
0402         $outdir/data/simulation/vertices_ttbar_hist.root \
0403         $refdir/simulation/vertices_ttbar_hist.root \
0404         "Vertices ttbar" \
0405         simulation/vertices_ttbar.html \
0406         simulation/vertices_ttbar_plots
0407 }
0408
0409 if [[ "$mode" == "all" || "$mode" == "simulation" ]]; then
0410     simulation fatras
0411     simulation geant4
0412
0413     generation
0414 fi
0415
0416 if [[ "$mode" == "all" || "$mode" == "kf" ]]; then
0417     run_histcmp \
0418         $outdir/data/trackfitting_kf/performance_trackfitting.root \
0419         $refdir/trackfitting_kf/performance_trackfitting.root \
0420         "Truth tracking (KF)" \
0421         trackfitting_kf/performance_trackfitting.html \
0422         trackfitting_kf/performance_trackfitting_plots \
0423         --config CI/physmon/config/trackfitting_kf.yml
0424 fi
0425
0426 if [[ "$mode" == "all" || "$mode" == "gsf" ]]; then
0427     run_histcmp \
0428         $outdir/data/trackfitting_gsf/performance_trackfitting.root \
0429         $refdir/trackfitting_gsf/performance_trackfitting.root \
0430         "Truth tracking (GSF)" \
0431         trackfitting_gsf/performance_trackfitting.html \
0432         trackfitting_gsf/performance_trackfitting_plots \
0433         --config CI/physmon/config/trackfitting_gsf.yml
0434 fi
0435
0436 if [[ "$mode" == "all" || "$mode" == "gx2f" ]]; then
0437     run_histcmp \
0438         $outdir/data/trackfitting_gx2f/performance_trackfitting.root \
0439         $refdir/trackfitting_gx2f/performance_trackfitting.root \
0440         "Truth tracking (GX2F)" \
0441         trackfitting_gx2f/performance_trackfitting.html \
0442         trackfitting_gx2f/performance_trackfitting_plots \
0443         --config CI/physmon/config/trackfitting_gx2f.yml
0444 fi
0445
0446 if [[ "$mode" == "all" || "$mode" == "refit_kf" ]]; then
0447     run_histcmp \
0448         $outdir/data/trackrefitting_kf/performance_trackrefitting.root \
0449         $refdir/trackrefitting_kf/performance_trackrefitting.root \
0450         "Truth tracking (KF refit)" \
0451         trackrefitting_kf/performance_trackrefitting.html \
0452         trackrefitting_kf/performance_trackrefitting_plots \
0453         --config CI/physmon/config/trackfitting_kf.yml
0454 fi
0455
0456 if [[ "$mode" == "all" || "$mode" == "refit_gsf" ]]; then
0457     run_histcmp \
0458         $outdir/data/trackrefitting_gsf/performance_trackrefitting.root \
0459         $refdir/trackrefitting_gsf/performance_trackrefitting.root \
0460         "Truth tracking (GSF refit)" \
0461         trackrefitting_gsf/performance_trackrefitting.html \
0462         trackrefitting_gsf/performance_trackrefitting_plots \
0463         --config CI/physmon/config/trackfitting_gsf.yml
0464 fi
0465
0466 if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
0467     trackfinding "trackfinding | single muon | truth smeared seeding" trackfinding_1muon/truth_smeared
0468     trackfinding "trackfinding | single muon | truth estimated seeding" trackfinding_1muon/truth_estimated
0469     trackfinding "trackfinding | single muon | default seeding" trackfinding_1muon/seeded
0470     trackfinding "trackfinding | single muon | orthogonal seeding" trackfinding_1muon/orthogonal
0471
0472     trackfinding "trackfinding | 4 muon x 50 vertices | default seeding" trackfinding_4muon_50vertices
0473     vertexing "trackfinding | 4 muon x 50 vertices | default seeding" trackfinding_4muon_50vertices CI/physmon/config/vertexing_4muon_50vertices.yml
0474
0475     trackfinding "trackfinding | ttbar with 200 pileup | default seeding" trackfinding_ttbar_pu200
0476     vertexing "trackfinding | ttbar with 200 pileup | default seeding" trackfinding_ttbar_pu200 CI/physmon/config/vertexing_ttbar_pu200.yml
0477 fi
0478
0479 if [[ "$mode" == "all" || "$mode" == "gx2f_vs_kf" ]]; then
0480     run_histcmp \
0481         $outdir/data/trackfitting_gx2f_vs_kf/performance_trackfitting_gx2f.root \
0482         $outdir/data/trackfitting_gx2f_vs_kf/performance_trackfitting_kf.root \
0483         "Comparison - Truth tracking (GX2F vs KF)" \
0484         trackfitting_gx2f_vs_kf/performance_trackfitting.html \
0485         trackfitting_gx2f_vs_kf/performance_trackfitting_plots \
0486         --config CI/physmon/config/info_only.yml \
0487         --label-reference=KF \
0488         --label-monitored=GX2F
0489 fi
0490
0491 run python3 CI/physmon/summary.py $histcmp_results \
0492   --md $outdir/summary.md \
0493   --html $outdir/summary.html
0494 ec=$(($ec | $?))
0495
0496 exit $ec