simulation_campaign_hepmc3/scripts/run.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # Load job environment (mask secrets)
0007 if ls environment*.sh ; then
0008   grep -v BEARER environment*.sh
0009   source environment*.sh
0010 fi
0011
0012 # Check arguments
0013 if [ $# -lt 1 ] ; then
0014   echo "Usage: "
0015   echo "  $0 <input> [n_chunk=10000] [i_chunk=]"
0016   echo
0017   echo "A typical npsim run requires from 0.5 to 5 core-seconds per event,"
0018   echo "and uses under 3 GB of memory. The output ROOT file for"
0019   echo "10k events take up about 2 GB in disk space."
0020   exit
0021 fi
0022
0023 # Startup
0024 echo "date sys: $(date)"
0025 echo "date web: $(date -d "$(curl --insecure --head --silent --max-redirs 0 google.com 2>&1 | grep Date: | cut -d' ' -f2-7)")"
0026 echo "hostname: $(hostname -f)"
0027 echo "uname:    $(uname -a)"
0028 echo "whoami:   $(whoami)"
0029 echo "pwd:      $(pwd)"
0030 echo "site:     ${GLIDEIN_Site:-}"
0031 echo "resource: ${GLIDEIN_ResourceName:-}"
0032 echo "http_proxy: ${http_proxy:-}"
0033 df -h --exclude-type=fuse --exclude-type=tmpfs
0034 ls -al
0035 test -f .job.ad && cat .job.ad
0036 test -f .machine.ad && cat .machine.ad
0037
0038 # Load container environment (include ${DETECTOR_VERSION})
0039 export DETECTOR_CONFIG_REQUESTED=${DETECTOR_CONFIG:-}
0040 export DETECTOR_VERSION_REQUESTED=${DETECTOR_VERSION:-main}
0041 source /opt/detector/epic-${DETECTOR_VERSION_REQUESTED}/bin/thisepic.sh
0042 export DETECTOR_VERSION=${DETECTOR_VERSION_REQUESTED}
0043 export DETECTOR_CONFIG=${DETECTOR_CONFIG_REQUESTED:-${DETECTOR_CONFIG:-$DETECTOR}}
0044 export SCRIPT_DIR=$(realpath $(dirname $0))
0045 export RUCIO_CONFIG=$SCRIPT_DIR/rucio.cfg
0046
0047 # Argument parsing
0048 # - input file basename
0049 BASENAME=${1}
0050 # - input file extension to determine type of simulation
0051 EXTENSION=${2}
0052 # - number of events
0053 EVENTS_PER_TASK=${3:-10000}
0054 # - current chunk (zero-based)
0055 if [ ${#} -lt 4 ] ; then
0056   TASK=""
0057   SEED=1
0058   SKIP_N_EVENTS=0
0059 else
0060   # 10-base input task number to 4-zero-padded task number
0061   TASK=".${4}"
0062   SEED=$((10#${4}+1))
0063   # assumes zero-based task number, can be zero-padded
0064   SKIP_N_EVENTS=$((10#${4}*EVENTS_PER_TASK))
0065 fi
0066
0067 # Output location
0068 BASEDIR=${DATADIR:-${PWD}}
0069
0070 # XRD Write locations (allow for empty URL override)
0071 XRDWURL=${XRDWURL-"xroots://dtn2201.jlab.org/"}
0072 XRDWBASE=${XRDWBASE:-"/eic/eic2/EPIC"}
0073
0074 # XRD Read locations (allow for empty URL override)
0075 XRDRURL=${XRDRURL-"root://dtn-eic.jlab.org/"}
0076 XRDRBASE=${XRDRBASE:-"/work/eic2/EPIC"}
0077
0078 # Local temp dir
0079 echo "SLURM_TMPDIR=${SLURM_TMPDIR:-}"
0080 echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}"
0081 echo "SLURM_ARRAY_JOB_ID=${SLURM_ARRAY_JOB_ID:-}"
0082 echo "SLURM_ARRAY_TASK_ID=${SLURM_ARRAY_TASK_ID:-}"
0083 echo "_CONDOR_SCRATCH_DIR=${_CONDOR_SCRATCH_DIR:-}"
0084 echo "OSG_WN_TMP=${OSG_WN_TMP:-}"
0085 if [ -n "${_CONDOR_SCRATCH_DIR:-}" ] ; then
0086   TMPDIR=${_CONDOR_SCRATCH_DIR}
0087 elif [ -n "${SLURM_TMPDIR:-}" ] ; then
0088   TMPDIR=${SLURM_TMPDIR}
0089 else
0090   if [ -d "/scratch/slurm/${SLURM_JOB_ID:-}" ] ; then
0091     TMPDIR="/scratch/slurm/${SLURM_JOB_ID:-}"
0092   else
0093     TMPDIR=${TMPDIR:-/tmp}/${$}
0094   fi
0095 fi
0096 echo "TMPDIR=${TMPDIR}"
0097 mkdir -p ${TMPDIR}
0098 ls -al ${TMPDIR}
0099
0100 # Input file parsing
0101 INPUT_FILE=${BASENAME}.${EXTENSION}
0102 TASKNAME=${TAG_SUFFIX:+${TAG_SUFFIX}_}$(basename ${BASENAME})${TASK}
0103 INPUT_DIR=$(dirname $(realpath --canonicalize-missing --relative-to=${BASEDIR} ${INPUT_FILE}))
0104 # - file.hepmc              -> TAG="", and avoid double // in S3 location
0105 # - EVGEN/file.hepmc        -> TAG="", and avoid double // in S3 location
0106 # - EVGEN/DIS/file.hepmc    -> TAG="DIS"
0107 # - EVGEN/DIS/NC/file.hepmc -> TAG="DIS/NC"
0108 # - ../file.hepmc           -> error
0109 if [ ! "${INPUT_DIR/\.\.\//}" = "${INPUT_DIR}" ] ; then
0110   echo "Error: Input file must be below current directory."
0111   exit
0112 fi
0113 INPUT_PREFIX=${INPUT_DIR/\/*/}
0114 TAG=${INPUT_DIR/${INPUT_PREFIX}\//}
0115 INPUT_DIR=${BASEDIR}/EVGEN/${TAG}
0116 mkdir -p ${INPUT_DIR}
0117 TAG=${DETECTOR_VERSION:-main}/${DETECTOR_CONFIG}/${TAG_PREFIX:+${TAG_PREFIX}/}${TAG}
0118
0119 if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0120   # Define location on xrootd from where to stream input file from
0121   INPUT_FILE=${XRDRURL}/${XRDRBASE}/${INPUT_FILE}
0122 else
0123   # Copy input file from xrootd
0124   xrdcp -f ${XRDRURL}/${XRDRBASE}/${INPUT_FILE} ${INPUT_DIR}
0125 fi
0126
0127 # Output file names
0128 LOG_DIR=LOG/${TAG}
0129 LOG_TEMP=${TMPDIR}/${LOG_DIR}
0130 mkdir -p ${LOG_TEMP}
0131 #
0132 FULL_DIR=FULL/${TAG}
0133 FULL_TEMP=${TMPDIR}/${FULL_DIR}
0134 mkdir -p ${FULL_TEMP}
0135 #
0136 RECO_DIR=RECO/${TAG}
0137 RECO_TEMP=${TMPDIR}/${RECO_DIR}
0138 mkdir -p ${RECO_TEMP}
0139
0140 # Run simulation
0141 {
0142   date
0143   eic-info
0144   # Common flags shared by both types of simulation
0145   common_flags=(
0146     --random.seed ${SEED:-1}
0147     --random.enableEventSeed
0148     --printLevel WARNING
0149     --filter.tracker 'edep0'
0150     --numberOfEvents ${EVENTS_PER_TASK}
0151     --outputFile ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0152   )
0153   # Uncommon flags based on EXTENSION
0154   if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0155     uncommon_flags=(
0156       --runType batch
0157       --skipNEvents ${SKIP_N_EVENTS}
0158       --hepmc3.useHepMC3 ${USEHEPMC3:-true}
0159       --compactFile ${DETECTOR_PATH}/${DETECTOR_CONFIG}${EBEAM:+${PBEAM:+_${EBEAM}x${PBEAM}}}.xml
0160       --inputFiles ${INPUT_FILE}
0161     )
0162   else
0163     uncommon_flags=(
0164       --runType run
0165       --enableGun
0166       --steeringFile ${INPUT_FILE}
0167       --compactFile ${DETECTOR_PATH}/${DETECTOR_CONFIG}.xml
0168     )
0169   fi
0170   # Run npsim with both common and uncommon flags
0171   prmon \
0172     --filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt \
0173     --json-summary ${LOG_TEMP}/${TASKNAME}.npsim.prmon.json \
0174     --log-filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.log \
0175     -- \
0176   npsim "${common_flags[@]}" "${uncommon_flags[@]}"
0177   ls -al ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0178 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.npsim.log | tail -n1000
0179
0180 # Run eicrecon reconstruction
0181 {
0182   date
0183   eic-info
0184   prmon \
0185     --filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt \
0186     --json-summary ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.json \
0187     --log-filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.log \
0188     -- \
0189   eicrecon \
0190     -Ppodio:output_file="${RECO_TEMP}/${TASKNAME}.eicrecon.tree.edm4eic.root" \
0191     -Pjana:warmup_timeout=0 -Pjana:timeout=0 \
0192     -Pplugins=janadot \
0193     "${FULL_TEMP}/${TASKNAME}.edm4hep.root"
0194   if [ -f jana.dot ] ; then mv jana.dot ${LOG_TEMP}/${TASKNAME}.eicrecon.dot ; fi
0195   ls -al ${RECO_TEMP}/${TASKNAME}.eicrecon.tree.edm4eic.root
0196 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.eicrecon.log | tail -n1000
0197
0198 # List log files
0199 ls -al ${LOG_TEMP}/${TASKNAME}.*
0200
0201 # Data egress to directory
0202
0203 if [ "${COPYLOG:-false}" == "true" ] ; then
0204   if [ "${USERUCIO:-false}" == "true" ] ; then
0205     python $SCRIPT_DIR/register_to_rucio.py \
0206     -f "${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt" "${LOG_TEMP}/${TASKNAME}.npsim.log" "${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt" "${LOG_TEMP}/${TASKNAME}.eicrecon.log" "${LOG_TEMP}/${TASKNAME}.eicrecon.dot" \
0207     -d "/${LOG_DIR}/${TASKNAME}.npsim.prmon.txt" "/${LOG_DIR}/${TASKNAME}.npsim.log" "/${LOG_DIR}/${TASKNAME}.eicrecon.prmon.txt" "/${LOG_DIR}/${TASKNAME}.eicrecon.log" "/${LOG_DIR}/${TASKNAME}.eicrecon.dot" \
0208     -s epic -r EIC-CLOUD-LOG
0209   else
0210     # Token for write authentication
0211     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0212     if [ -n ${XRDWURL} ] ; then
0213       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${LOG_DIR} || echo "Cannot write log outputs to xrootd server"
0214     else
0215       mkdir -p ${XRDWBASE}/${LOG_DIR} || echo "Cannot write log outputs to xrootd server"
0216     fi
0217     xrdcp --force --recursive ${LOG_TEMP}/${TASKNAME}.* ${XRDWURL}/${XRDWBASE}/${LOG_DIR}
0218   fi
0219 fi
0220
0221 if [ "${COPYFULL:-false}" == "true" ] ; then
0222   if [ "${USERUCIO:-false}" == "true" ] ; then
0223     python $SCRIPT_DIR/register_to_rucio.py -f "${FULL_TEMP}/${TASKNAME}.edm4hep.root" -d "/${FULL_DIR}/${TASKNAME}.edm4hep.root" -s epic -r EIC-XRD
0224   else
0225     # Token for write authentication
0226     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0227     if [ -n ${XRDWURL} ] ; then
0228       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${FULL_DIR} || echo "Cannot write simulation outputs to xrootd server"
0229     else
0230       mkdir -p ${XRDWBASE}/${FULL_DIR} || echo "Cannot write simulation outputs to xrootd server"
0231     fi
0232     xrdcp --force --recursive ${FULL_TEMP}/${TASKNAME}.edm4hep.root ${XRDWURL}/${XRDWBASE}/${FULL_DIR}
0233   fi
0234 fi
0235
0236 if [ "${COPYRECO:-false}" == "true" ] ; then
0237   if [ "${USERUCIO:-false}" == "true" ] ; then
0238     python $SCRIPT_DIR/register_to_rucio.py -f "${RECO_TEMP}/${TASKNAME}.eicrecon.tree.edm4eic.root" -d "/${RECO_DIR}/${TASKNAME}.eicrecon.tree.edm4eic.root" -s epic -r EIC-XRD
0239   else
0240     # Token for write authentication
0241     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0242     if [ -n ${XRDWURL} ] ; then
0243       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${RECO_DIR} || echo "Cannot write reconstructed outputs to xrootd server"
0244     else
0245       mkdir -p ${XRDWBASE}/${RECO_DIR} || echo "Cannot write reconstructed outputs to xrootd server"
0246     fi
0247     xrdcp --force --recursive ${RECO_TEMP}/${TASKNAME}*.edm4eic.root ${XRDWURL}/${XRDWBASE}/${RECO_DIR}
0248   fi
0249 fi
0250
0251 # closeout
0252 date
0253 find ${TMPDIR}
0254 du -sh ${TMPDIR}