simulation_campaign_hepmc3/scripts/run.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # Load bearer token or fall back to x509 proxy for xrootd authentication
0007 setup_xrd_auth() {
0008   echo "BEARER_TOKEN file location: ${_CONDOR_CREDS:-.}/eic.use"
0009   if [ -f "${_CONDOR_CREDS:-.}/eic.use" ]; then
0010     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0011     echo "BEARER_TOKEN loaded successfully"
0012   else
0013     echo "WARNING: BEARER_TOKEN file not found at ${_CONDOR_CREDS:-.}/eic.use"
0014     if [ -f "x509_user_proxy" ]; then
0015       echo "Found x509_user_proxy, setting X509_USER_PROXY"
0016       export X509_USER_PROXY="x509_user_proxy"
0017     fi
0018   fi
0019 }
0020
0021 # Load job environment (mask secrets)
0022 if ls environment*.sh ; then
0023   grep -v BEARER environment*.sh
0024   source environment*.sh
0025 fi
0026
0027 # Check arguments
0028 if [ $# -lt 1 ] ; then
0029   echo "Usage: "
0030   echo "  $0 <input> [n_chunk=10000] [i_chunk=]"
0031   echo
0032   echo "A typical npsim run requires from 0.5 to 5 core-seconds per event,"
0033   echo "and uses under 3 GB of memory. The output ROOT file for"
0034   echo "10k events take up about 2 GB in disk space."
0035   exit
0036 fi
0037
0038 # Startup
0039 echo "date sys: $(date)"
0040 echo "date web: $(date -d "$(curl --insecure --head --silent --max-redirs 0 google.com 2>&1 | grep Date: | cut -d' ' -f2-7)")"
0041 echo "hostname: $(hostname -f)"
0042 echo "uname:    $(uname -a)"
0043 echo "whoami:   $(whoami)"
0044 echo "pwd:      $(pwd)"
0045 echo "site:     ${GLIDEIN_Site:-}"
0046 echo "resource: ${GLIDEIN_ResourceName:-}"
0047 echo "http_proxy: ${http_proxy:-}"
0048 df -h --exclude-type=fuse --exclude-type=tmpfs
0049 ls -al
0050 test -f .job.ad && cat .job.ad
0051 test -f .machine.ad && cat .machine.ad
0052
0053 # Load container environment (include ${DETECTOR_VERSION})
0054 export DETECTOR_CONFIG_REQUESTED=${DETECTOR_CONFIG:-}
0055 export DETECTOR_VERSION_REQUESTED=${DETECTOR_VERSION:-main}
0056 source /opt/detector/epic-${DETECTOR_VERSION_REQUESTED}/bin/thisepic.sh
0057 export DETECTOR_VERSION=${DETECTOR_VERSION_REQUESTED}
0058 export DETECTOR_CONFIG=${DETECTOR_CONFIG_REQUESTED:-${DETECTOR_CONFIG:-$DETECTOR}}
0059 export SCRIPT_DIR=$(realpath $(dirname $0))
0060 export RUCIO_CONFIG=$SCRIPT_DIR/rucio.cfg
0061 export RUCIO_ACCOUNT=eicprod
0062
0063 # Print out the location of the rucio config file
0064 echo $RUCIO_CONFIG
0065
0066 # Argument parsing
0067 # - input file basename
0068 BASENAME=${1}
0069 # - input file extension to determine type of simulation
0070 EXTENSION=${2}
0071 # - number of events
0072 EVENTS_PER_TASK=${3:-10000}
0073 # - current chunk (zero-based)
0074 if [ ${#} -lt 4 ] ; then
0075   TASK=""
0076   SEED=1
0077   SKIP_N_EVENTS=0
0078 else
0079   # 10-base input task number to 4-zero-padded task number
0080   TASK=".${4}"
0081   SEED=$((10#${4}+1))
0082   # assumes zero-based task number, can be zero-padded
0083   SKIP_N_EVENTS=$((10#${4}*EVENTS_PER_TASK))
0084 fi
0085
0086 # Output location
0087 BASEDIR=${DATADIR:-${PWD}}
0088
0089 # XRD Write locations (allow for empty URL override)
0090 XRDWURL=${XRDWURL-"xroots://dtn2201.jlab.org/"}
0091 XRDWBASE=${XRDWBASE:-"/eic/eic2/EPIC"}
0092
0093 # XRD Read locations (allow for empty URL override)
0094 XRDRURL=${XRDRURL-"root://dtn-eic.jlab.org/"}
0095 XRDRBASE=${XRDRBASE:-"/volatile/eic/EPIC"}
0096
0097 # Local temp dir
0098 echo "SLURM_TMPDIR=${SLURM_TMPDIR:-}"
0099 echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}"
0100 echo "SLURM_ARRAY_JOB_ID=${SLURM_ARRAY_JOB_ID:-}"
0101 echo "SLURM_ARRAY_TASK_ID=${SLURM_ARRAY_TASK_ID:-}"
0102 echo "_CONDOR_SCRATCH_DIR=${_CONDOR_SCRATCH_DIR:-}"
0103 echo "OSG_WN_TMP=${OSG_WN_TMP:-}"
0104 if [ -n "${_CONDOR_SCRATCH_DIR:-}" ] ; then
0105   TMPDIR=${_CONDOR_SCRATCH_DIR}
0106 elif [ -n "${SLURM_TMPDIR:-}" ] ; then
0107   TMPDIR=${SLURM_TMPDIR}
0108 else
0109   if [ -d "/scratch/slurm/${SLURM_JOB_ID:-}" ] ; then
0110     TMPDIR="/scratch/slurm/${SLURM_JOB_ID:-}"
0111   else
0112     TMPDIR=${TMPDIR:-/tmp}/${$}
0113   fi
0114 fi
0115 echo "TMPDIR=${TMPDIR}"
0116 mkdir -p ${TMPDIR}
0117 ls -al ${TMPDIR}
0118
0119 # Input file parsing
0120 INPUT_FILE=${BASENAME}.${EXTENSION}
0121 TASKNAME=${TAG_SUFFIX:+${TAG_SUFFIX}_}$(basename ${BASENAME})${TASK}
0122 INPUT_DIR=$(dirname $(realpath --canonicalize-missing --relative-to=${BASEDIR} ${INPUT_FILE}))
0123 # - file.hepmc              -> TAG="", and avoid double // in S3 location
0124 # - EVGEN/file.hepmc        -> TAG="", and avoid double // in S3 location
0125 # - EVGEN/DIS/file.hepmc    -> TAG="DIS"
0126 # - EVGEN/DIS/NC/file.hepmc -> TAG="DIS/NC"
0127 # - ../file.hepmc           -> error
0128 if [ ! "${INPUT_DIR/\.\.\//}" = "${INPUT_DIR}" ] ; then
0129   echo "Error: Input file must be below current directory."
0130   exit
0131 fi
0132 INPUT_PREFIX=${INPUT_DIR/\/*/}
0133 TAG=${INPUT_DIR/${INPUT_PREFIX}\//}
0134 INPUT_DIR=${BASEDIR}/EVGEN/${TAG}
0135 mkdir -p ${INPUT_DIR}
0136 TAG=${DETECTOR_VERSION:-main}/${DETECTOR_CONFIG}/${TAG_PREFIX:+${TAG_PREFIX}/}${TAG}
0137
0138 if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0139   # Define location on xrootd from where to stream input file from
0140   INPUT_FILE=${XRDRURL}/${XRDRBASE}/${INPUT_FILE}
0141 else
0142   # Copy input file from xrootd
0143   xrdcp -f ${XRDRURL}/${XRDRBASE}/${INPUT_FILE} ${INPUT_DIR}
0144 fi
0145
0146 # Output file names
0147 LOG_DIR=LOG/${TAG}
0148 LOG_TEMP=${TMPDIR}/${LOG_DIR}
0149 mkdir -p ${LOG_TEMP}
0150 #
0151 FULL_DIR=FULL/${TAG}
0152 FULL_TEMP=${TMPDIR}/${FULL_DIR}
0153 mkdir -p ${FULL_TEMP}
0154 #
0155 RECO_DIR=RECO/${TAG}
0156 RECO_TEMP=${TMPDIR}/${RECO_DIR}
0157 mkdir -p ${RECO_TEMP}
0158
0159 # Mix background events if the input file is a hepmc file
0160 if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0161   BG_ARGS=()
0162
0163   SIGNAL_STATUS_VALUE=${SIGNAL_STATUS:-0}
0164   STABLE_STATUSES="$((${SIGNAL_STATUS_VALUE}+1))"
0165   DECAY_STATUSES="$((${SIGNAL_STATUS_VALUE}+2))"
0166
0167   if [[ -n "${BG_FILES:-}" ]]; then
0168     while read -r bg_file; do
0169       file=$(echo "$bg_file" | jq -r '.file')
0170       freq=$(echo "$bg_file" | jq -r '.freq')
0171       skip=$(echo "$bg_file" | jq -r '.skip')
0172
0173       # This ensures that the number of background events skipped before sampling from the source is atleast 1.
0174       skip=$(awk "BEGIN {print int((${SKIP_N_EVENTS}*${skip})+1)}")
0175       status=$(echo "$bg_file" | jq -r '.status')
0176       BG_ARGS+=(--bgFile "$file" "$freq" "$skip" "$status")
0177       STABLE_STATUSES="${STABLE_STATUSES} $((status+1))"
0178       DECAY_STATUSES="${DECAY_STATUSES} $((status+2))"
0179     done < <(jq -c '.[]' ${BG_FILES})
0180     # Run the background merger with proper logging
0181     {
0182       date
0183       eic-info
0184       prmon \
0185         --filename ${LOG_TEMP}/${TASKNAME}.hepmcmerger.prmon.txt \
0186         --json-summary ${LOG_TEMP}/${TASKNAME}.hepmcmerger.prmon.json \
0187         -- \
0188       SignalBackgroundMerger \
0189         --rngSeed ${SEED:-1} \
0190         --nSlices ${EVENTS_PER_TASK} \
0191         --signalSkip ${SKIP_N_EVENTS} \
0192         --signalFile ${INPUT_FILE} \
0193         --signalFreq ${SIGNAL_FREQ:-0} \
0194         --signalStatus ${SIGNAL_STATUS:-0} \
0195         "${BG_ARGS[@]}" \
0196         --outputFile ${FULL_TEMP}/${TASKNAME}.hepmc3.tree.root
0197
0198     } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.hepmcmerger.log | tail -n1000
0199
0200     # Use background merged file as input for next stage
0201     INPUT_FILE=${FULL_TEMP}/${TASKNAME}.hepmc3.tree.root
0202     # Don't skip events on the background merged file
0203     SKIP_N_EVENTS=0
0204   else
0205     echo "No background mixing will be performed since no sources are provided"
0206   fi
0207 else
0208   echo "No background mixing is performed for singles"
0209 fi
0210
0211 # Run simulation
0212 {
0213   date
0214   eic-info
0215   # Common flags shared by both types of simulation
0216   common_flags=(
0217     --random.seed ${SEED:-1}
0218     --random.enableEventSeed
0219     --printLevel WARNING
0220     --filter.tracker 'edep0'
0221     --numberOfEvents ${EVENTS_PER_TASK}
0222     --compactFile ${DETECTOR_PATH}/${DETECTOR_CONFIG}${EBEAM:+${PBEAM:+_${EBEAM}x${PBEAM}}}.xml
0223     --outputFile ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0224   )
0225   # Uncommon flags based on EXTENSION
0226   if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0227     uncommon_flags=(
0228       --runType batch
0229       --skipNEvents ${SKIP_N_EVENTS}
0230       --hepmc3.useHepMC3 ${USEHEPMC3:-true}
0231       --physics.alternativeStableStatuses "${STABLE_STATUSES}"
0232       --physics.alternativeDecayStatuses "${DECAY_STATUSES}"
0233       --inputFiles ${INPUT_FILE}
0234     )
0235   else
0236     uncommon_flags=(
0237       --runType run
0238       --enableGun
0239       --steeringFile ${INPUT_FILE}
0240     )
0241   fi
0242   # Run npsim with both common and uncommon flags
0243   prmon \
0244     --filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt \
0245     --json-summary ${LOG_TEMP}/${TASKNAME}.npsim.prmon.json \
0246     --log-filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.log \
0247     -- \
0248   npsim "${common_flags[@]}" "${uncommon_flags[@]}"
0249   ls -al ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0250 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.npsim.log | tail -n1000
0251
0252 # Run eicrecon reconstruction
0253 {
0254   date
0255   eic-info
0256   prmon \
0257     --filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt \
0258     --json-summary ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.json \
0259     --log-filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.log \
0260     -- \
0261   eicrecon \
0262     -Pdd4hep:xml_files="${DETECTOR_PATH}/${DETECTOR_CONFIG}${EBEAM:+${PBEAM:+_${EBEAM}x${PBEAM}}}.xml" \
0263     -Ppodio:output_file="${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root" \
0264     -Pjana:warmup_timeout=0 -Pjana:timeout=0 \
0265     -Pplugins=janadot \
0266     "${FULL_TEMP}/${TASKNAME}.edm4hep.root"
0267   if [ -f jana.dot ] ; then mv jana.dot ${LOG_TEMP}/${TASKNAME}.eicrecon.dot ; fi
0268   ls -al ${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root
0269 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.eicrecon.log | tail -n1000
0270
0271 # List log files
0272 ls -al ${LOG_TEMP}/${TASKNAME}.*
0273
0274 # Build metadata JSON string for Rucio registration
0275 # Extract software release from eic-info: strip trailing (-default)?-<40hexchars> in one pass
0276 JUG_XL_TAG=$(eic-info 2>/dev/null | grep -oP '(?<=jug_dev: )([\d.]+-(?=stable)|.*?\K)(stable|unstable|nightly|default)')
0277 # Extract metadata from FULL file via podio (all fields except software_release)
0278 PODIO_ARGS=("${FULL_TEMP}/${TASKNAME}.edm4hep.root")
0279 if [[ "$EXTENSION" != "hepmc3.tree.root" ]]; then
0280   PODIO_ARGS+=(--gun)
0281 fi
0282 if [[ "${BASENAME}" == *"BACKGROUNDS"* ]]; then
0283   PODIO_ARGS+=(--no-beam)
0284 fi
0285 PODIO_JSON=$(python $SCRIPT_DIR/parse_podio_metadata.py "${PODIO_ARGS[@]}")
0286
0287 # Only software_release remains outside podio scope
0288 METADATA_JSON_BASE=$(jq -n --arg software_release "${JUG_XL_TAG}" '{software_release: $software_release}')
0289
0290 # Merge podio-extracted fields (geometry_config, data_level, beam/gun params)
0291 METADATA_JSON_FULL=$(jq -n --argjson base "${METADATA_JSON_BASE}" --argjson podio "${PODIO_JSON}" '$base * $podio')
0292 METADATA_JSON_RECO=$(jq -n --argjson base "${METADATA_JSON_FULL}" '$base | .data_level = "reconstruction"')
0293
0294 # Data egress to directory
0295
0296 if [ "${COPYLOG:-false}" == "true" ] ; then
0297   if [ "${USERUCIO:-false}" == "true" ] ; then
0298     TIME_TAG=$(date --iso-8601=second)
0299     TARFILE="${LOG_TEMP}/${TASKNAME}.log.tar.gz"
0300
0301     # Initialize an empty array to hold existing files
0302     FILES_TO_TAR=()
0303
0304     # List of expected files
0305     for FILE in \
0306       "${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt" \
0307       "${LOG_TEMP}/${TASKNAME}.npsim.log" \
0308       "${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt" \
0309       "${LOG_TEMP}/${TASKNAME}.eicrecon.log" \
0310       "${LOG_TEMP}/${TASKNAME}.eicrecon.dot" \
0311       "${LOG_TEMP}/${TASKNAME}.hepmcmerger.log"
0312     do
0313       if [ -f "$FILE" ]; then
0314         FILES_TO_TAR+=("$FILE")
0315       fi
0316     done
0317
0318     # Create the tar archive only if there are files to include
0319     if [ ${#FILES_TO_TAR[@]} -gt 0 ]; then
0320       tar -czvf "$TARFILE" "${FILES_TO_TAR[@]}"
0321     else
0322       echo "No log files found to archive."
0323     fi
0324
0325     python $SCRIPT_DIR/register_to_rucio.py \
0326     -f "${LOG_TEMP}/${TASKNAME}.log.tar.gz" \
0327     -d "/${LOG_DIR}/${TASKNAME}.${TIME_TAG}.log.tar.gz" \
0328     -s epic -r ${LOG_RSE:-EIC-XRD-LOG} --noregister
0329   else
0330     echo "=== DEBUG: Attempting to copy LOG files to xrootd ==="
0331     setup_xrd_auth
0332     echo "Source: ${LOG_TEMP}/${TASKNAME}.*"
0333     echo "Destination: ${XRDWURL}/${XRDWBASE}/${LOG_DIR}"
0334     if [ -n ${XRDWURL} ] ; then
0335       echo "Creating directory: xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${LOG_DIR}"
0336       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${LOG_DIR} || echo "ERROR: Cannot create log directory on xrootd server"
0337     fi
0338     echo "Running: xrdcp --debug 2 --force --recursive ${LOG_TEMP}/${TASKNAME}.* ${XRDWURL}/${XRDWBASE}/${LOG_DIR}"
0339     xrdcp --debug 2 --force --recursive ${LOG_TEMP}/${TASKNAME}.* ${XRDWURL}/${XRDWBASE}/${LOG_DIR} || echo "ERROR: xrdcp failed with exit code $?"
0340     echo "=== DEBUG: LOG copy attempt completed ==="
0341   fi
0342 fi
0343
0344 if [ "${COPYFULL:-false}" == "true" ] ; then
0345   # Validate ROOT file before transfer
0346   echo "=== Validating FULL ROOT file before transfer ==="
0347   python $SCRIPT_DIR/validate_rootfile.py "${FULL_TEMP}/${TASKNAME}.edm4hep.root"
0348   if [ $? -ne 0 ]; then
0349     echo "ERROR: FULL ROOT file validation failed. Skipping transfer."
0350     exit 65
0351   fi
0352   echo "FULL ROOT file validation passed."
0353
0354   if [ "${USERUCIO:-false}" == "true" ] ; then
0355     python $SCRIPT_DIR/register_to_rucio.py -f "${FULL_TEMP}/${TASKNAME}.edm4hep.root" -d "/${FULL_DIR}/${TASKNAME}.edm4hep.root" -s epic -r ${OUT_RSE:-EIC-XRD} --metadata-json "${METADATA_JSON_FULL}" || { echo "ERROR: Rucio registration failed for FULL file."; exit 78; }
0356   else
0357     echo "=== DEBUG: Attempting to copy FULL files to xrootd ==="
0358     setup_xrd_auth
0359     echo "Source: ${FULL_TEMP}/${TASKNAME}.edm4hep.root"
0360     echo "Destination: ${XRDWURL}/${XRDWBASE}/${FULL_DIR}"
0361     if [ -n ${XRDWURL} ] ; then
0362       echo "Creating directory: xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${FULL_DIR}"
0363       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${FULL_DIR} || echo "ERROR: Cannot create simulation directory on xrootd server"
0364     fi
0365     echo "Running: xrdcp --debug 2 --force --recursive ${FULL_TEMP}/${TASKNAME}.edm4hep.root ${XRDWURL}/${XRDWBASE}/${FULL_DIR}"
0366     xrdcp --debug 2 --force --recursive ${FULL_TEMP}/${TASKNAME}.edm4hep.root ${XRDWURL}/${XRDWBASE}/${FULL_DIR} || echo "ERROR: xrdcp failed with exit code $?"
0367     echo "=== DEBUG: FULL copy attempt completed ==="
0368   fi
0369 fi
0370
0371 if [ "${COPYRECO:-false}" == "true" ] ; then
0372   # Validate ROOT file before transfer
0373   echo "=== Validating RECO ROOT file before transfer ==="
0374   python $SCRIPT_DIR/validate_rootfile.py "${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root"
0375   if [ $? -ne 0 ]; then
0376     echo "ERROR: RECO ROOT file validation failed. Skipping transfer."
0377     exit 65
0378   fi
0379   echo "RECO ROOT file validation passed."
0380
0381   if [ "${USERUCIO:-false}" == "true" ] ; then
0382     python $SCRIPT_DIR/register_to_rucio.py -f "${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root" -d "/${RECO_DIR}/${TASKNAME}.eicrecon.edm4eic.root" -s epic -r ${OUT_RSE:-EIC-XRD} --metadata-json "${METADATA_JSON_RECO}" || { echo "ERROR: Rucio registration failed for RECO file."; exit 78; }
0383   else
0384     echo "=== DEBUG: Attempting to copy RECO files to xrootd ==="
0385     setup_xrd_auth
0386     echo "Source: ${RECO_TEMP}/${TASKNAME}*.edm4eic.root"
0387     echo "Destination: ${XRDWURL}/${XRDWBASE}/${RECO_DIR}"
0388     if [ -n ${XRDWURL} ] ; then
0389       echo "Creating directory: xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${RECO_DIR}"
0390       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${RECO_DIR} || echo "ERROR: Cannot create reconstruction directory on xrootd server"
0391     fi
0392     echo "Running: xrdcp --debug 2 --force --recursive ${RECO_TEMP}/${TASKNAME}*.edm4eic.root ${XRDWURL}/${XRDWBASE}/${RECO_DIR}"
0393     xrdcp --debug 2 --force --recursive ${RECO_TEMP}/${TASKNAME}*.edm4eic.root ${XRDWURL}/${XRDWBASE}/${RECO_DIR} || echo "ERROR: xrdcp failed with exit code $?"
0394     echo "=== DEBUG: RECO copy attempt completed ==="
0395   fi
0396 fi
0397
0398 # closeout
0399 date
0400 find ${TMPDIR}
0401 du -sh ${TMPDIR}