simulation_campaign_hepmc3/scripts/run.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # Load job environment (mask secrets)
0007 if ls environment*.sh ; then
0008   grep -v BEARER environment*.sh
0009   source environment*.sh
0010 fi
0011
0012 # Check arguments
0013 if [ $# -lt 1 ] ; then
0014   echo "Usage: "
0015   echo "  $0 <input> [n_chunk=10000] [i_chunk=]"
0016   echo
0017   echo "A typical npsim run requires from 0.5 to 5 core-seconds per event,"
0018   echo "and uses under 3 GB of memory. The output ROOT file for"
0019   echo "10k events take up about 2 GB in disk space."
0020   exit
0021 fi
0022
0023 # Startup
0024 echo "date sys: $(date)"
0025 echo "date web: $(date -d "$(curl --insecure --head --silent --max-redirs 0 google.com 2>&1 | grep Date: | cut -d' ' -f2-7)")"
0026 echo "hostname: $(hostname -f)"
0027 echo "uname:    $(uname -a)"
0028 echo "whoami:   $(whoami)"
0029 echo "pwd:      $(pwd)"
0030 echo "site:     ${GLIDEIN_Site:-}"
0031 echo "resource: ${GLIDEIN_ResourceName:-}"
0032 echo "http_proxy: ${http_proxy:-}"
0033 df -h --exclude-type=fuse --exclude-type=tmpfs
0034 ls -al
0035 test -f .job.ad && cat .job.ad
0036 test -f .machine.ad && cat .machine.ad
0037
0038 # Load container environment (include ${DETECTOR_VERSION})
0039 export DETECTOR_CONFIG_REQUESTED=${DETECTOR_CONFIG:-}
0040 export DETECTOR_VERSION_REQUESTED=${DETECTOR_VERSION:-main}
0041 source /opt/detector/epic-${DETECTOR_VERSION_REQUESTED}/bin/thisepic.sh
0042 export DETECTOR_VERSION=${DETECTOR_VERSION_REQUESTED}
0043 export DETECTOR_CONFIG=${DETECTOR_CONFIG_REQUESTED:-${DETECTOR_CONFIG:-$DETECTOR}}
0044 export SCRIPT_DIR=$(realpath $(dirname $0))
0045 export RUCIO_CONFIG=$SCRIPT_DIR/rucio.cfg
0046 export RUCIO_ACCOUNT=eicprod
0047
0048 # Print out the location of the rucio config file
0049 echo $RUCIO_CONFIG
0050
0051 # Argument parsing
0052 # - input file basename
0053 BASENAME=${1}
0054 # - input file extension to determine type of simulation
0055 EXTENSION=${2}
0056 # - number of events
0057 EVENTS_PER_TASK=${3:-10000}
0058 # - current chunk (zero-based)
0059 if [ ${#} -lt 4 ] ; then
0060   TASK=""
0061   SEED=1
0062   SKIP_N_EVENTS=0
0063 else
0064   # 10-base input task number to 4-zero-padded task number
0065   TASK=".${4}"
0066   SEED=$((10#${4}+1))
0067   # assumes zero-based task number, can be zero-padded
0068   SKIP_N_EVENTS=$((10#${4}*EVENTS_PER_TASK))
0069 fi
0070
0071 # Output location
0072 BASEDIR=${DATADIR:-${PWD}}
0073
0074 # XRD Write locations (allow for empty URL override)
0075 XRDWURL=${XRDWURL-"xroots://dtn2201.jlab.org/"}
0076 XRDWBASE=${XRDWBASE:-"/eic/eic2/EPIC"}
0077
0078 # XRD Read locations (allow for empty URL override)
0079 XRDRURL=${XRDRURL-"root://dtn-eic.jlab.org/"}
0080 XRDRBASE=${XRDRBASE:-"/volatile/eic/EPIC"}
0081
0082 # Local temp dir
0083 echo "SLURM_TMPDIR=${SLURM_TMPDIR:-}"
0084 echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}"
0085 echo "SLURM_ARRAY_JOB_ID=${SLURM_ARRAY_JOB_ID:-}"
0086 echo "SLURM_ARRAY_TASK_ID=${SLURM_ARRAY_TASK_ID:-}"
0087 echo "_CONDOR_SCRATCH_DIR=${_CONDOR_SCRATCH_DIR:-}"
0088 echo "OSG_WN_TMP=${OSG_WN_TMP:-}"
0089 if [ -n "${_CONDOR_SCRATCH_DIR:-}" ] ; then
0090   TMPDIR=${_CONDOR_SCRATCH_DIR}
0091 elif [ -n "${SLURM_TMPDIR:-}" ] ; then
0092   TMPDIR=${SLURM_TMPDIR}
0093 else
0094   if [ -d "/scratch/slurm/${SLURM_JOB_ID:-}" ] ; then
0095     TMPDIR="/scratch/slurm/${SLURM_JOB_ID:-}"
0096   else
0097     TMPDIR=${TMPDIR:-/tmp}/${$}
0098   fi
0099 fi
0100 echo "TMPDIR=${TMPDIR}"
0101 mkdir -p ${TMPDIR}
0102 ls -al ${TMPDIR}
0103
0104 # Input file parsing
0105 INPUT_FILE=${BASENAME}.${EXTENSION}
0106 TASKNAME=${TAG_SUFFIX:+${TAG_SUFFIX}_}$(basename ${BASENAME})${TASK}
0107 INPUT_DIR=$(dirname $(realpath --canonicalize-missing --relative-to=${BASEDIR} ${INPUT_FILE}))
0108 # - file.hepmc              -> TAG="", and avoid double // in S3 location
0109 # - EVGEN/file.hepmc        -> TAG="", and avoid double // in S3 location
0110 # - EVGEN/DIS/file.hepmc    -> TAG="DIS"
0111 # - EVGEN/DIS/NC/file.hepmc -> TAG="DIS/NC"
0112 # - ../file.hepmc           -> error
0113 if [ ! "${INPUT_DIR/\.\.\//}" = "${INPUT_DIR}" ] ; then
0114   echo "Error: Input file must be below current directory."
0115   exit
0116 fi
0117 INPUT_PREFIX=${INPUT_DIR/\/*/}
0118 TAG=${INPUT_DIR/${INPUT_PREFIX}\//}
0119 INPUT_DIR=${BASEDIR}/EVGEN/${TAG}
0120 mkdir -p ${INPUT_DIR}
0121 TAG=${DETECTOR_VERSION:-main}/${DETECTOR_CONFIG}/${TAG_PREFIX:+${TAG_PREFIX}/}${TAG}
0122
0123 if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0124   # Define location on xrootd from where to stream input file from
0125   INPUT_FILE=${XRDRURL}/${XRDRBASE}/${INPUT_FILE}
0126 else
0127   # Copy input file from xrootd
0128   xrdcp -f ${XRDRURL}/${XRDRBASE}/${INPUT_FILE} ${INPUT_DIR}
0129 fi
0130
0131 # Output file names
0132 LOG_DIR=LOG/${TAG}
0133 LOG_TEMP=${TMPDIR}/${LOG_DIR}
0134 mkdir -p ${LOG_TEMP}
0135 #
0136 FULL_DIR=FULL/${TAG}
0137 FULL_TEMP=${TMPDIR}/${FULL_DIR}
0138 mkdir -p ${FULL_TEMP}
0139 #
0140 RECO_DIR=RECO/${TAG}
0141 RECO_TEMP=${TMPDIR}/${RECO_DIR}
0142 mkdir -p ${RECO_TEMP}
0143
0144 # Mix background events if the input file is a hepmc file
0145 if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0146   BG_ARGS=()
0147
0148   SIGNAL_STATUS_VALUE=${SIGNAL_STATUS:-0}
0149   STABLE_STATUSES="$((${SIGNAL_STATUS_VALUE}+1))"
0150   DECAY_STATUSES="$((${SIGNAL_STATUS_VALUE}+2))"
0151
0152   if [[ -n "${BG_FILES:-}" ]]; then
0153     while read -r bg_file; do
0154       file=$(echo "$bg_file" | jq -r '.file')
0155       freq=$(echo "$bg_file" | jq -r '.freq')
0156       skip=$(echo "$bg_file" | jq -r '.skip')
0157
0158       # This ensures that the number of background events skipped before sampling from the source is atleast 1.
0159       skip=$(awk "BEGIN {print int((${SKIP_N_EVENTS}*${skip})+1)}")
0160       status=$(echo "$bg_file" | jq -r '.status')
0161       BG_ARGS+=(--bgFile "$file" "$freq" "$skip" "$status")
0162       STABLE_STATUSES="${STABLE_STATUSES} $((status+1))"
0163       DECAY_STATUSES="${DECAY_STATUSES} $((status+2))"
0164     done < <(jq -c '.[]' ${BG_FILES})
0165     # Run the background merger with proper logging
0166     {
0167       date
0168       eic-info
0169       prmon \
0170         --filename ${LOG_TEMP}/${TASKNAME}.hepmcmerger.prmon.txt \
0171         --json-summary ${LOG_TEMP}/${TASKNAME}.hepmcmerger.prmon.json \
0172         -- \
0173       SignalBackgroundMerger \
0174         --rngSeed ${SEED:-1} \
0175         --nSlices ${EVENTS_PER_TASK} \
0176         --signalSkip ${SKIP_N_EVENTS} \
0177         --signalFile ${INPUT_FILE} \
0178         --signalFreq ${SIGNAL_FREQ:-0} \
0179         --signalStatus ${SIGNAL_STATUS:-0} \
0180         "${BG_ARGS[@]}" \
0181         --outputFile ${FULL_TEMP}/${TASKNAME}.hepmc3.tree.root
0182
0183     } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.hepmcmerger.log | tail -n1000
0184
0185     # Use background merged file as input for next stage
0186     INPUT_FILE=${FULL_TEMP}/${TASKNAME}.hepmc3.tree.root
0187     # Don't skip events on the background merged file
0188     SKIP_N_EVENTS=0
0189   else
0190     echo "No background mixing will be performed since no sources are provided"
0191   fi
0192 else
0193   echo "No background mixing is performed for singles"
0194 fi
0195
0196 # Run simulation
0197 {
0198   date
0199   eic-info
0200   # Common flags shared by both types of simulation
0201   common_flags=(
0202     --random.seed ${SEED:-1}
0203     --random.enableEventSeed
0204     --printLevel WARNING
0205     --filter.tracker 'edep0'
0206     --numberOfEvents ${EVENTS_PER_TASK}
0207     --compactFile ${DETECTOR_PATH}/${DETECTOR_CONFIG}${EBEAM:+${PBEAM:+_${EBEAM}x${PBEAM}}}.xml
0208     --outputFile ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0209   )
0210   # Uncommon flags based on EXTENSION
0211   if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0212     uncommon_flags=(
0213       --runType batch
0214       --skipNEvents ${SKIP_N_EVENTS}
0215       --hepmc3.useHepMC3 ${USEHEPMC3:-true}
0216       --physics.alternativeStableStatuses "${STABLE_STATUSES}"
0217       --physics.alternativeDecayStatuses "${DECAY_STATUSES}"
0218       --inputFiles ${INPUT_FILE}
0219     )
0220   else
0221     uncommon_flags=(
0222       --runType run
0223       --enableGun
0224       --steeringFile ${INPUT_FILE}
0225     )
0226   fi
0227   # Run npsim with both common and uncommon flags
0228   prmon \
0229     --filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt \
0230     --json-summary ${LOG_TEMP}/${TASKNAME}.npsim.prmon.json \
0231     --log-filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.log \
0232     -- \
0233   npsim "${common_flags[@]}" "${uncommon_flags[@]}"
0234   ls -al ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0235 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.npsim.log | tail -n1000
0236
0237 # Run eicrecon reconstruction
0238 {
0239   date
0240   eic-info
0241   prmon \
0242     --filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt \
0243     --json-summary ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.json \
0244     --log-filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.log \
0245     -- \
0246   eicrecon \
0247     -Pdd4hep:xml_files="${DETECTOR_PATH}/${DETECTOR_CONFIG}${EBEAM:+${PBEAM:+_${EBEAM}x${PBEAM}}}.xml" \
0248     -Ppodio:output_file="${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root" \
0249     -Pjana:warmup_timeout=0 -Pjana:timeout=0 \
0250     -Pplugins=janadot \
0251     "${FULL_TEMP}/${TASKNAME}.edm4hep.root"
0252   if [ -f jana.dot ] ; then mv jana.dot ${LOG_TEMP}/${TASKNAME}.eicrecon.dot ; fi
0253   ls -al ${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root
0254 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.eicrecon.log | tail -n1000
0255
0256 # List log files
0257 ls -al ${LOG_TEMP}/${TASKNAME}.*
0258
0259 # Data egress to directory
0260
0261 if [ "${COPYLOG:-false}" == "true" ] ; then
0262   if [ "${USERUCIO:-false}" == "true" ] ; then
0263     TIME_TAG=$(date --iso-8601=second)
0264     TARFILE="${LOG_TEMP}/${TASKNAME}.log.tar.gz"
0265
0266     # Initialize an empty array to hold existing files
0267     FILES_TO_TAR=()
0268
0269     # List of expected files
0270     for FILE in \
0271       "${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt" \
0272       "${LOG_TEMP}/${TASKNAME}.npsim.log" \
0273       "${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt" \
0274       "${LOG_TEMP}/${TASKNAME}.eicrecon.log" \
0275       "${LOG_TEMP}/${TASKNAME}.eicrecon.dot" \
0276       "${LOG_TEMP}/${TASKNAME}.hepmcmerger.log"
0277     do
0278       if [ -f "$FILE" ]; then
0279         FILES_TO_TAR+=("$FILE")
0280       fi
0281     done
0282
0283     # Create the tar archive only if there are files to include
0284     if [ ${#FILES_TO_TAR[@]} -gt 0 ]; then
0285       tar -czvf "$TARFILE" "${FILES_TO_TAR[@]}"
0286     else
0287       echo "No log files found to archive."
0288     fi
0289
0290     python $SCRIPT_DIR/register_to_rucio.py \
0291     -f "${LOG_TEMP}/${TASKNAME}.log.tar.gz" \
0292     -d "/${LOG_DIR}/${TASKNAME}.${TIME_TAG}.log.tar.gz" \
0293     -s epic -r EIC-XRD-LOG -noregister
0294   else
0295     # Token for write authentication
0296     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0297     if [ -n ${XRDWURL} ] ; then
0298       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${LOG_DIR} || echo "Cannot write log outputs to xrootd server"
0299     else
0300       mkdir -p ${XRDWBASE}/${LOG_DIR} || echo "Cannot write log outputs to xrootd server"
0301     fi
0302     xrdcp --force --recursive ${LOG_TEMP}/${TASKNAME}.* ${XRDWURL}/${XRDWBASE}/${LOG_DIR}
0303   fi
0304 fi
0305
0306 if [ "${COPYFULL:-false}" == "true" ] ; then
0307   if [ "${USERUCIO:-false}" == "true" ] ; then
0308     python $SCRIPT_DIR/register_to_rucio.py -f "${FULL_TEMP}/${TASKNAME}.edm4hep.root" -d "/${FULL_DIR}/${TASKNAME}.edm4hep.root" -s epic -r EIC-XRD
0309   else
0310     # Token for write authentication
0311     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0312     if [ -n ${XRDWURL} ] ; then
0313       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${FULL_DIR} || echo "Cannot write simulation outputs to xrootd server"
0314     else
0315       mkdir -p ${XRDWBASE}/${FULL_DIR} || echo "Cannot write simulation outputs to xrootd server"
0316     fi
0317     xrdcp --force --recursive ${FULL_TEMP}/${TASKNAME}.edm4hep.root ${XRDWURL}/${XRDWBASE}/${FULL_DIR}
0318   fi
0319 fi
0320
0321 if [ "${COPYRECO:-false}" == "true" ] ; then
0322   if [ "${USERUCIO:-false}" == "true" ] ; then
0323     python $SCRIPT_DIR/register_to_rucio.py -f "${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root" -d "/${RECO_DIR}/${TASKNAME}.eicrecon.edm4eic.root" -s epic -r EIC-XRD
0324   else
0325     # Token for write authentication
0326     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0327     if [ -n ${XRDWURL} ] ; then
0328       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${RECO_DIR} || echo "Cannot write reconstructed outputs to xrootd server"
0329     else
0330       mkdir -p ${XRDWBASE}/${RECO_DIR} || echo "Cannot write reconstructed outputs to xrootd server"
0331     fi
0332     xrdcp --force --recursive ${RECO_TEMP}/${TASKNAME}*.edm4eic.root ${XRDWURL}/${XRDWBASE}/${RECO_DIR}
0333   fi
0334 fi
0335
0336 # closeout
0337 date
0338 find ${TMPDIR}
0339 du -sh ${TMPDIR}