simulation_campaign_hepmc3/scripts/run.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # Load job environment (mask secrets)
0007 if ls environment*.sh ; then
0008   grep -v BEARER environment*.sh
0009   source environment*.sh
0010 fi
0011
0012 # Check arguments
0013 if [ $# -lt 1 ] ; then
0014   echo "Usage: "
0015   echo "  $0 <input> [n_chunk=10000] [i_chunk=]"
0016   echo
0017   echo "A typical npsim run requires from 0.5 to 5 core-seconds per event,"
0018   echo "and uses under 3 GB of memory. The output ROOT file for"
0019   echo "10k events take up about 2 GB in disk space."
0020   exit
0021 fi
0022
0023 # Startup
0024 echo "date sys: $(date)"
0025 echo "date web: $(date -d "$(curl --insecure --head --silent --max-redirs 0 google.com 2>&1 | grep Date: | cut -d' ' -f2-7)")"
0026 echo "hostname: $(hostname -f)"
0027 echo "uname:    $(uname -a)"
0028 echo "whoami:   $(whoami)"
0029 echo "pwd:      $(pwd)"
0030 echo "site:     ${GLIDEIN_Site:-}"
0031 echo "resource: ${GLIDEIN_ResourceName:-}"
0032 echo "http_proxy: ${http_proxy:-}"
0033 df -h --exclude-type=fuse --exclude-type=tmpfs
0034 ls -al
0035 test -f .job.ad && cat .job.ad
0036 test -f .machine.ad && cat .machine.ad
0037
0038 # Load container environment (include ${DETECTOR_VERSION})
0039 export DETECTOR_CONFIG_REQUESTED=${DETECTOR_CONFIG:-}
0040 export DETECTOR_VERSION_REQUESTED=${DETECTOR_VERSION:-main}
0041 source /opt/detector/epic-${DETECTOR_VERSION_REQUESTED}/bin/thisepic.sh
0042 export DETECTOR_VERSION=${DETECTOR_VERSION_REQUESTED}
0043 export DETECTOR_CONFIG=${DETECTOR_CONFIG_REQUESTED:-${DETECTOR_CONFIG:-$DETECTOR}}
0044 export SCRIPT_DIR=$(realpath $(dirname $0))
0045 export RUCIO_CONFIG=$SCRIPT_DIR/rucio.cfg
0046
0047 # Argument parsing
0048 # - input file basename
0049 BASENAME=${1}
0050 # - input file extension to determine type of simulation
0051 EXTENSION=${2}
0052 # - number of events
0053 EVENTS_PER_TASK=${3:-10000}
0054 # - current chunk (zero-based)
0055 if [ ${#} -lt 4 ] ; then
0056   TASK=""
0057   SEED=1
0058   SKIP_N_EVENTS=0
0059 else
0060   # 10-base input task number to 4-zero-padded task number
0061   TASK=".${4}"
0062   SEED=$((10#${4}+1))
0063   # assumes zero-based task number, can be zero-padded
0064   SKIP_N_EVENTS=$((10#${4}*EVENTS_PER_TASK))
0065 fi
0066
0067 # Output location
0068 BASEDIR=${DATADIR:-${PWD}}
0069
0070 # XRD Write locations (allow for empty URL override)
0071 XRDWURL=${XRDWURL-"xroots://dtn2201.jlab.org/"}
0072 XRDWBASE=${XRDWBASE:-"/eic/eic2/EPIC"}
0073
0074 # XRD Read locations (allow for empty URL override)
0075 XRDRURL=${XRDRURL-"root://dtn-eic.jlab.org/"}
0076 XRDRBASE=${XRDRBASE:-"/volatile/eic/EPIC"}
0077
0078 # Local temp dir
0079 echo "SLURM_TMPDIR=${SLURM_TMPDIR:-}"
0080 echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}"
0081 echo "SLURM_ARRAY_JOB_ID=${SLURM_ARRAY_JOB_ID:-}"
0082 echo "SLURM_ARRAY_TASK_ID=${SLURM_ARRAY_TASK_ID:-}"
0083 echo "_CONDOR_SCRATCH_DIR=${_CONDOR_SCRATCH_DIR:-}"
0084 echo "OSG_WN_TMP=${OSG_WN_TMP:-}"
0085 if [ -n "${_CONDOR_SCRATCH_DIR:-}" ] ; then
0086   TMPDIR=${_CONDOR_SCRATCH_DIR}
0087 elif [ -n "${SLURM_TMPDIR:-}" ] ; then
0088   TMPDIR=${SLURM_TMPDIR}
0089 else
0090   if [ -d "/scratch/slurm/${SLURM_JOB_ID:-}" ] ; then
0091     TMPDIR="/scratch/slurm/${SLURM_JOB_ID:-}"
0092   else
0093     TMPDIR=${TMPDIR:-/tmp}/${$}
0094   fi
0095 fi
0096 echo "TMPDIR=${TMPDIR}"
0097 mkdir -p ${TMPDIR}
0098 ls -al ${TMPDIR}
0099
0100 # Input file parsing
0101 INPUT_FILE=${BASENAME}.${EXTENSION}
0102 TASKNAME=${TAG_SUFFIX:+${TAG_SUFFIX}_}$(basename ${BASENAME})${TASK}
0103 INPUT_DIR=$(dirname $(realpath --canonicalize-missing --relative-to=${BASEDIR} ${INPUT_FILE}))
0104 # - file.hepmc              -> TAG="", and avoid double // in S3 location
0105 # - EVGEN/file.hepmc        -> TAG="", and avoid double // in S3 location
0106 # - EVGEN/DIS/file.hepmc    -> TAG="DIS"
0107 # - EVGEN/DIS/NC/file.hepmc -> TAG="DIS/NC"
0108 # - ../file.hepmc           -> error
0109 if [ ! "${INPUT_DIR/\.\.\//}" = "${INPUT_DIR}" ] ; then
0110   echo "Error: Input file must be below current directory."
0111   exit
0112 fi
0113 INPUT_PREFIX=${INPUT_DIR/\/*/}
0114 TAG=${INPUT_DIR/${INPUT_PREFIX}\//}
0115 INPUT_DIR=${BASEDIR}/EVGEN/${TAG}
0116 mkdir -p ${INPUT_DIR}
0117 TAG=${DETECTOR_VERSION:-main}/${DETECTOR_CONFIG}/${TAG_PREFIX:+${TAG_PREFIX}/}${TAG}
0118
0119 if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0120   # Define location on xrootd from where to stream input file from
0121   INPUT_FILE=${XRDRURL}/${XRDRBASE}/${INPUT_FILE}
0122 else
0123   # Copy input file from xrootd
0124   xrdcp -f ${XRDRURL}/${XRDRBASE}/${INPUT_FILE} ${INPUT_DIR}
0125 fi
0126
0127 # Output file names
0128 LOG_DIR=LOG/${TAG}
0129 LOG_TEMP=${TMPDIR}/${LOG_DIR}
0130 mkdir -p ${LOG_TEMP}
0131 #
0132 FULL_DIR=FULL/${TAG}
0133 FULL_TEMP=${TMPDIR}/${FULL_DIR}
0134 mkdir -p ${FULL_TEMP}
0135 #
0136 RECO_DIR=RECO/${TAG}
0137 RECO_TEMP=${TMPDIR}/${RECO_DIR}
0138 mkdir -p ${RECO_TEMP}
0139
0140 # Mix background events if the input file is a hepmc file
0141 if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0142   BG_ARGS=()
0143
0144   SIGNAL_STATUS_VALUE=${SIGNAL_STATUS:-0}
0145   STABLE_STATUSES="$((${SIGNAL_STATUS_VALUE}+1))"
0146   DECAY_STATUSES="$((${SIGNAL_STATUS_VALUE}+2))"
0147
0148   if [[ -n "${BG_FILES:-}" ]]; then
0149     while read -r bg_file; do
0150       file=$(echo "$bg_file" | jq -r '.file')
0151       freq=$(echo "$bg_file" | jq -r '.freq')
0152       skip=$(echo "$bg_file" | jq -r '.skip')
0153
0154       # This ensures that the number of background events skipped before sampling from the source is atleast 1.
0155       skip=$(awk "BEGIN {print int((${SKIP_N_EVENTS}*${skip})+1)}")
0156       status=$(echo "$bg_file" | jq -r '.status')
0157       BG_ARGS+=(--bgFile "$file" "$freq" "$skip" "$status")
0158       STABLE_STATUSES="${STABLE_STATUSES} $((status+1))"
0159       DECAY_STATUSES="${DECAY_STATUSES} $((status+2))"
0160     done < <(jq -c '.[]' ${BG_FILES})
0161   else
0162     echo "No background mixing will be performed since no sources are provided"
0163   fi
0164
0165   # Run the background merger with proper logging
0166   {
0167     date
0168     eic-info
0169     prmon \
0170       --filename ${LOG_TEMP}/${TASKNAME}.hepmcmerger.prmon.txt \
0171       --json-summary ${LOG_TEMP}/${TASKNAME}.hepmcmerger.prmon.json \
0172       -- \
0173     SignalBackgroundMerger \
0174       --rngSeed ${SEED:-1} \
0175       --nSlices ${EVENTS_PER_TASK} \
0176       --signalSkip ${SKIP_N_EVENTS} \
0177       --signalFile ${INPUT_FILE} \
0178       --signalFreq ${SIGNAL_FREQ:-0} \
0179       --signalStatus ${SIGNAL_STATUS:-0} \
0180       "${BG_ARGS[@]}" \
0181       --outputFile ${FULL_TEMP}/${TASKNAME}.hepmc3.tree.root
0182
0183   } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.hepmcmerger.log | tail -n1000
0184
0185   # Use background merged file as input for next stage
0186   INPUT_FILE=${FULL_TEMP}/${TASKNAME}.hepmc3.tree.root
0187   # Don't skip events on the background merged file
0188   SKIP_N_EVENTS=0
0189 else
0190   echo "No background mixing is performed for singles"
0191 fi
0192
0193 # Run simulation
0194 {
0195   date
0196   eic-info
0197   # Common flags shared by both types of simulation
0198   common_flags=(
0199     --random.seed ${SEED:-1}
0200     --random.enableEventSeed
0201     --printLevel WARNING
0202     --filter.tracker 'edep0'
0203     --numberOfEvents ${EVENTS_PER_TASK}
0204     --compactFile ${DETECTOR_PATH}/${DETECTOR_CONFIG}${EBEAM:+${PBEAM:+_${EBEAM}x${PBEAM}}}.xml
0205     --outputFile ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0206   )
0207   # Uncommon flags based on EXTENSION
0208   if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0209     uncommon_flags=(
0210       --runType batch
0211       --skipNEvents ${SKIP_N_EVENTS}
0212       --hepmc3.useHepMC3 ${USEHEPMC3:-true}
0213       --physics.alternativeStableStatuses "${STABLE_STATUSES}"
0214       --physics.alternativeDecayStatuses "${DECAY_STATUSES}"
0215       --inputFiles ${INPUT_FILE}
0216     )
0217   else
0218     uncommon_flags=(
0219       --runType run
0220       --enableGun
0221       --steeringFile ${INPUT_FILE}
0222     )
0223   fi
0224   # Run npsim with both common and uncommon flags
0225   prmon \
0226     --filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt \
0227     --json-summary ${LOG_TEMP}/${TASKNAME}.npsim.prmon.json \
0228     --log-filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.log \
0229     -- \
0230   npsim "${common_flags[@]}" "${uncommon_flags[@]}"
0231   ls -al ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0232 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.npsim.log | tail -n1000
0233
0234 # Run eicrecon reconstruction
0235 {
0236   date
0237   eic-info
0238   prmon \
0239     --filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt \
0240     --json-summary ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.json \
0241     --log-filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.log \
0242     -- \
0243   eicrecon \
0244     -Pdd4hep:xml_files="${DETECTOR_PATH}/${DETECTOR_CONFIG}${EBEAM:+${PBEAM:+_${EBEAM}x${PBEAM}}}.xml" \
0245     -Ppodio:output_file="${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root" \
0246     -Pjana:warmup_timeout=0 -Pjana:timeout=0 \
0247     -Pplugins=janadot \
0248     "${FULL_TEMP}/${TASKNAME}.edm4hep.root"
0249   if [ -f jana.dot ] ; then mv jana.dot ${LOG_TEMP}/${TASKNAME}.eicrecon.dot ; fi
0250   ls -al ${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root
0251 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.eicrecon.log | tail -n1000
0252
0253 # List log files
0254 ls -al ${LOG_TEMP}/${TASKNAME}.*
0255
0256 # Data egress to directory
0257
0258 if [ "${COPYLOG:-false}" == "true" ] ; then
0259   if [ "${USERUCIO:-false}" == "true" ] ; then
0260     TIME_TAG=$(date --iso-8601=second)
0261     TARFILE="${LOG_TEMP}/${TASKNAME}.log.tar.gz"
0262
0263     # Initialize an empty array to hold existing files
0264     FILES_TO_TAR=()
0265
0266     # List of expected files
0267     for FILE in \
0268       "${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt" \
0269       "${LOG_TEMP}/${TASKNAME}.npsim.log" \
0270       "${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt" \
0271       "${LOG_TEMP}/${TASKNAME}.eicrecon.log" \
0272       "${LOG_TEMP}/${TASKNAME}.eicrecon.dot" \
0273       "${LOG_TEMP}/${TASKNAME}.hepmcmerger.log"
0274     do
0275       if [ -f "$FILE" ]; then
0276         FILES_TO_TAR+=("$FILE")
0277       fi
0278     done
0279
0280     # Create the tar archive only if there are files to include
0281     if [ ${#FILES_TO_TAR[@]} -gt 0 ]; then
0282       tar -czvf "$TARFILE" "${FILES_TO_TAR[@]}"
0283     else
0284       echo "No log files found to archive."
0285     fi
0286
0287     python $SCRIPT_DIR/register_to_rucio.py \
0288     -f "${LOG_TEMP}/${TASKNAME}.log.tar.gz" \
0289     -d "/${LOG_DIR}/${TASKNAME}.${TIME_TAG}.log.tar.gz" \
0290     -s epic -r EIC-XRD-LOG -noregister
0291   else
0292     # Token for write authentication
0293     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0294     if [ -n ${XRDWURL} ] ; then
0295       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${LOG_DIR} || echo "Cannot write log outputs to xrootd server"
0296     else
0297       mkdir -p ${XRDWBASE}/${LOG_DIR} || echo "Cannot write log outputs to xrootd server"
0298     fi
0299     xrdcp --force --recursive ${LOG_TEMP}/${TASKNAME}.* ${XRDWURL}/${XRDWBASE}/${LOG_DIR}
0300   fi
0301 fi
0302
0303 if [ "${COPYFULL:-false}" == "true" ] ; then
0304   if [ "${USERUCIO:-false}" == "true" ] ; then
0305     python $SCRIPT_DIR/register_to_rucio.py -f "${FULL_TEMP}/${TASKNAME}.edm4hep.root" -d "/${FULL_DIR}/${TASKNAME}.edm4hep.root" -s epic -r EIC-XRD
0306   else
0307     # Token for write authentication
0308     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0309     if [ -n ${XRDWURL} ] ; then
0310       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${FULL_DIR} || echo "Cannot write simulation outputs to xrootd server"
0311     else
0312       mkdir -p ${XRDWBASE}/${FULL_DIR} || echo "Cannot write simulation outputs to xrootd server"
0313     fi
0314     xrdcp --force --recursive ${FULL_TEMP}/${TASKNAME}.edm4hep.root ${XRDWURL}/${XRDWBASE}/${FULL_DIR}
0315   fi
0316 fi
0317
0318 if [ "${COPYRECO:-false}" == "true" ] ; then
0319   if [ "${USERUCIO:-false}" == "true" ] ; then
0320     python $SCRIPT_DIR/register_to_rucio.py -f "${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root" -d "/${RECO_DIR}/${TASKNAME}.eicrecon.edm4eic.root" -s epic -r EIC-XRD
0321   else
0322     # Token for write authentication
0323     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0324     if [ -n ${XRDWURL} ] ; then
0325       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${RECO_DIR} || echo "Cannot write reconstructed outputs to xrootd server"
0326     else
0327       mkdir -p ${XRDWBASE}/${RECO_DIR} || echo "Cannot write reconstructed outputs to xrootd server"
0328     fi
0329     xrdcp --force --recursive ${RECO_TEMP}/${TASKNAME}*.edm4eic.root ${XRDWURL}/${XRDWBASE}/${RECO_DIR}
0330   fi
0331 fi
0332
0333 # closeout
0334 date
0335 find ${TMPDIR}
0336 du -sh ${TMPDIR}