simulation_campaign_hepmc3/scripts/run.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # Load bearer token or fall back to x509 proxy for xrootd authentication
0007 setup_xrd_auth() {
0008   echo "BEARER_TOKEN file location: ${_CONDOR_CREDS:-.}/eic.use"
0009   if [ -f "${_CONDOR_CREDS:-.}/eic.use" ]; then
0010     export BEARER_TOKEN=$(cat ${_CONDOR_CREDS:-.}/eic.use)
0011     echo "BEARER_TOKEN loaded successfully"
0012   else
0013     echo "WARNING: BEARER_TOKEN file not found at ${_CONDOR_CREDS:-.}/eic.use"
0014     if [ -f "x509_user_proxy" ]; then
0015       echo "Found x509_user_proxy, setting X509_USER_PROXY"
0016       export X509_USER_PROXY="x509_user_proxy"
0017     fi
0018   fi
0019 }
0020
0021 # Load job environment (mask secrets)
0022 if ls environment*.sh ; then
0023   grep -v BEARER environment*.sh
0024   source environment*.sh
0025 fi
0026
0027 # Check arguments
0028 if [ $# -lt 1 ] ; then
0029   echo "Usage: "
0030   echo "  $0 <input> [n_chunk=10000] [i_chunk=]"
0031   echo
0032   echo "A typical npsim run requires from 0.5 to 5 core-seconds per event,"
0033   echo "and uses under 3 GB of memory. The output ROOT file for"
0034   echo "10k events take up about 2 GB in disk space."
0035   exit
0036 fi
0037
0038 # Startup
0039 echo "date sys: $(date)"
0040 echo "date web: $(date -d "$(curl --insecure --head --silent --max-redirs 0 google.com 2>&1 | grep Date: | cut -d' ' -f2-7)")"
0041 echo "hostname: $(hostname -f)"
0042 echo "uname:    $(uname -a)"
0043 echo "whoami:   $(whoami)"
0044 echo "pwd:      $(pwd)"
0045 echo "site:     ${GLIDEIN_Site:-}"
0046 echo "resource: ${GLIDEIN_ResourceName:-}"
0047 echo "http_proxy: ${http_proxy:-}"
0048 df -h --exclude-type=fuse --exclude-type=tmpfs
0049 ls -al
0050 test -f .job.ad && cat .job.ad
0051 test -f .machine.ad && cat .machine.ad
0052
0053 # Load container environment (include ${DETECTOR_VERSION})
0054 export DETECTOR_CONFIG_REQUESTED=${DETECTOR_CONFIG:-}
0055 export DETECTOR_VERSION_REQUESTED=${DETECTOR_VERSION:-main}
0056 source /opt/detector/epic-${DETECTOR_VERSION_REQUESTED}/bin/thisepic.sh
0057 export DETECTOR_VERSION=${DETECTOR_VERSION_REQUESTED}
0058 export DETECTOR_CONFIG=${DETECTOR_CONFIG_REQUESTED:-${DETECTOR_CONFIG:-$DETECTOR}}
0059 export SCRIPT_DIR=$(realpath $(dirname $0))
0060 export RUCIO_CONFIG=$SCRIPT_DIR/rucio.cfg
0061 export RUCIO_ACCOUNT=eicprod
0062
0063 # Print out the location of the rucio config file
0064 echo $RUCIO_CONFIG
0065
0066 # Argument parsing
0067 # - input file basename
0068 BASENAME=${1}
0069 # - input file extension to determine type of simulation
0070 EXTENSION=${2}
0071 # - number of events
0072 EVENTS_PER_TASK=${3:-10000}
0073 # - current chunk (zero-based)
0074 if [ ${#} -lt 4 ] ; then
0075   TASK=""
0076   SEED=1
0077   SKIP_N_EVENTS=0
0078 else
0079   # 10-base input task number to 4-zero-padded task number
0080   TASK=".${4}"
0081   SEED=$((10#${4}+1))
0082   # assumes zero-based task number, can be zero-padded
0083   SKIP_N_EVENTS=$((10#${4}*EVENTS_PER_TASK))
0084 fi
0085
0086 # Output location
0087 BASEDIR=${DATADIR:-${PWD}}
0088
0089 # XRD Write locations (allow for empty URL override)
0090 XRDWURL=${XRDWURL-"xroots://dtn2201.jlab.org/"}
0091 XRDWBASE=${XRDWBASE:-"/eic/eic2/EPIC"}
0092
0093 # XRD Read locations (allow for empty URL override)
0094 XRDRURL=${XRDRURL-"root://dtn-eic.jlab.org/"}
0095 XRDRBASE=${XRDRBASE:-"/volatile/eic/EPIC"}
0096
0097 # Local temp dir
0098 echo "SLURM_TMPDIR=${SLURM_TMPDIR:-}"
0099 echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}"
0100 echo "SLURM_ARRAY_JOB_ID=${SLURM_ARRAY_JOB_ID:-}"
0101 echo "SLURM_ARRAY_TASK_ID=${SLURM_ARRAY_TASK_ID:-}"
0102 echo "_CONDOR_SCRATCH_DIR=${_CONDOR_SCRATCH_DIR:-}"
0103 echo "OSG_WN_TMP=${OSG_WN_TMP:-}"
0104 if [ -n "${_CONDOR_SCRATCH_DIR:-}" ] ; then
0105   TMPDIR=${_CONDOR_SCRATCH_DIR}
0106 elif [ -n "${SLURM_TMPDIR:-}" ] ; then
0107   TMPDIR=${SLURM_TMPDIR}
0108 else
0109   if [ -d "/scratch/slurm/${SLURM_JOB_ID:-}" ] ; then
0110     TMPDIR="/scratch/slurm/${SLURM_JOB_ID:-}"
0111   else
0112     TMPDIR=${TMPDIR:-/tmp}/${$}
0113   fi
0114 fi
0115 echo "TMPDIR=${TMPDIR}"
0116 mkdir -p ${TMPDIR}
0117 ls -al ${TMPDIR}
0118
0119 # Input file parsing
0120 INPUT_FILE=${BASENAME}.${EXTENSION}
0121 TASKNAME=${TAG_SUFFIX:+${TAG_SUFFIX}_}$(basename ${BASENAME})${TASK}
0122 INPUT_DIR=$(dirname $(realpath --canonicalize-missing --relative-to=${BASEDIR} ${INPUT_FILE}))
0123 # - file.hepmc              -> TAG="", and avoid double // in S3 location
0124 # - EVGEN/file.hepmc        -> TAG="", and avoid double // in S3 location
0125 # - EVGEN/DIS/file.hepmc    -> TAG="DIS"
0126 # - EVGEN/DIS/NC/file.hepmc -> TAG="DIS/NC"
0127 # - ../file.hepmc           -> error
0128 if [ ! "${INPUT_DIR/\.\.\//}" = "${INPUT_DIR}" ] ; then
0129   echo "Error: Input file must be below current directory."
0130   exit
0131 fi
0132 INPUT_PREFIX=${INPUT_DIR/\/*/}
0133 TAG=${INPUT_DIR/${INPUT_PREFIX}\//}
0134 INPUT_DIR=${BASEDIR}/EVGEN/${TAG}
0135 mkdir -p ${INPUT_DIR}
0136 TAG=${DETECTOR_VERSION:-main}/${DETECTOR_CONFIG}/${TAG_PREFIX:+${TAG_PREFIX}/}${TAG}
0137
0138 if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0139   # Define location on xrootd from where to stream input file from
0140   INPUT_FILE=${XRDRURL}/${XRDRBASE}/${INPUT_FILE}
0141 else
0142   # Copy input file from xrootd
0143   xrdcp -f ${XRDRURL}/${XRDRBASE}/${INPUT_FILE} ${INPUT_DIR}
0144 fi
0145
0146 # Output file names
0147 LOG_DIR=LOG/${TAG}
0148 LOG_TEMP=${TMPDIR}/${LOG_DIR}
0149 mkdir -p ${LOG_TEMP}
0150 #
0151 FULL_DIR=FULL/${TAG}
0152 FULL_TEMP=${TMPDIR}/${FULL_DIR}
0153 mkdir -p ${FULL_TEMP}
0154 #
0155 RECO_DIR=RECO/${TAG}
0156 RECO_TEMP=${TMPDIR}/${RECO_DIR}
0157 mkdir -p ${RECO_TEMP}
0158
0159 # Integration window (ns) used with bg freq (kHz) to compute per-event skip
0160 INTEGRATION_WINDOW=${INTEGRATION_WINDOW:-2000}
0161
0162 # Mix background events if the input file is a hepmc file
0163 if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0164   BG_ARGS=()
0165
0166   SIGNAL_STATUS_VALUE=${SIGNAL_STATUS:-0}
0167   STABLE_STATUSES="$((${SIGNAL_STATUS_VALUE}+1))"
0168   DECAY_STATUSES="$((${SIGNAL_STATUS_VALUE}+2))"
0169
0170   if [[ -n "${BG_FILES:-}" ]]; then
0171     while read -r bg_file; do
0172       file=$(echo "$bg_file" | jq -r '.file')
0173       freq=$(echo "$bg_file" | jq -r '.freq')
0174
0175       # bg events per signal event = freq[kHz]*1e3 * INTEGRATION_WINDOW[ns]*1e-9 = freq*window*1e-6
0176       skip=$(awk "BEGIN {print (${freq})*(${INTEGRATION_WINDOW})*1e-6}")
0177
0178       # Mix BASENAME hash into seed so different signal files decorrelate, not just task index.
0179       # Shift hash into high bits so it can't cancel with task-index low bits (2^20 task headroom).
0180       # sha256 truncated to 32 bits — better distribution than cksum/CRC for similar filenames.
0181       BASENAME_HASH=$(( 16#$(printf '%s' "${BASENAME}" | sha256sum | cut -c1-8) ))
0182       MIXED_SEED=$(( (BASENAME_HASH << 20) + SEED ))
0183
0184       # Rate-scaled advance + seed-driven random offset. Rate term preserves bg-per-signal proportionality;
0185       # random term decorrelates parallel tasks. Merger wraps bg file, so large offsets are safe.
0186       skip=$(awk "BEGIN {srand(${MIXED_SEED}); print int((${SKIP_N_EVENTS}*${skip})+1) + int(rand()*2147483647)}")
0187       status=$(echo "$bg_file" | jq -r '.status')
0188       BG_ARGS+=(--bgFile "$file" "$freq" "$skip" "$status")
0189       STABLE_STATUSES="${STABLE_STATUSES} $((status+1))"
0190       DECAY_STATUSES="${DECAY_STATUSES} $((status+2))"
0191     done < <(jq -c '.[]' ${BG_FILES})
0192     # Run the background merger with proper logging
0193     {
0194       date
0195       eic-info
0196       prmon \
0197         --filename ${LOG_TEMP}/${TASKNAME}.hepmcmerger.prmon.txt \
0198         --json-summary ${LOG_TEMP}/${TASKNAME}.hepmcmerger.prmon.json \
0199         -- \
0200       SignalBackgroundMerger \
0201         --rngSeed ${SEED:-1} \
0202         --nSlices ${EVENTS_PER_TASK} \
0203         --signalSkip ${SKIP_N_EVENTS} \
0204         --signalFile ${INPUT_FILE} \
0205         --signalFreq ${SIGNAL_FREQ:-0} \
0206         --signalStatus ${SIGNAL_STATUS:-0} \
0207         --intWindow ${INTEGRATION_WINDOW} \
0208         "${BG_ARGS[@]}" \
0209         --outputFile ${FULL_TEMP}/${TASKNAME}.hepmc3.tree.root
0210
0211     } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.hepmcmerger.log | tail -n1000
0212
0213     # Use background merged file as input for next stage
0214     INPUT_FILE=${FULL_TEMP}/${TASKNAME}.hepmc3.tree.root
0215     # Don't skip events on the background merged file
0216     SKIP_N_EVENTS=0
0217   else
0218     echo "No background mixing will be performed since no sources are provided"
0219   fi
0220 else
0221   echo "No background mixing is performed for singles"
0222 fi
0223
0224 # Run simulation
0225 {
0226   date
0227   eic-info
0228   # Common flags shared by both types of simulation
0229   common_flags=(
0230     --random.seed ${SEED:-1}
0231     --random.enableEventSeed
0232     --printLevel WARNING
0233     --filter.tracker 'edep0'
0234     --numberOfEvents ${EVENTS_PER_TASK}
0235     --compactFile ${DETECTOR_PATH}/${DETECTOR_CONFIG}${EBEAM:+${PBEAM:+_${EBEAM}x${PBEAM}}}.xml
0236     --outputFile ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0237   )
0238   # Uncommon flags based on EXTENSION
0239   if [[ "$EXTENSION" == "hepmc3.tree.root" ]]; then
0240     uncommon_flags=(
0241       --runType batch
0242       --skipNEvents ${SKIP_N_EVENTS}
0243       --hepmc3.useHepMC3 ${USEHEPMC3:-true}
0244       --physics.alternativeStableStatuses "${STABLE_STATUSES}"
0245       --physics.alternativeDecayStatuses "${DECAY_STATUSES}"
0246       --inputFiles ${INPUT_FILE}
0247     )
0248   else
0249     uncommon_flags=(
0250       --runType run
0251       --enableGun
0252       --steeringFile ${INPUT_FILE}
0253     )
0254   fi
0255   # Run npsim with both common and uncommon flags
0256   prmon \
0257     --filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt \
0258     --json-summary ${LOG_TEMP}/${TASKNAME}.npsim.prmon.json \
0259     --log-filename ${LOG_TEMP}/${TASKNAME}.npsim.prmon.log \
0260     -- \
0261   npsim "${common_flags[@]}" "${uncommon_flags[@]}"
0262   ls -al ${FULL_TEMP}/${TASKNAME}.edm4hep.root
0263 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.npsim.log | tail -n1000
0264
0265 # Run eicrecon reconstruction
0266 {
0267   date
0268   eic-info
0269   prmon \
0270     --filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt \
0271     --json-summary ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.json \
0272     --log-filename ${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.log \
0273     -- \
0274   eicrecon \
0275     -Pdd4hep:xml_files="${DETECTOR_PATH}/${DETECTOR_CONFIG}${EBEAM:+${PBEAM:+_${EBEAM}x${PBEAM}}}.xml" \
0276     -Ppodio:output_file="${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root" \
0277     -Pjana:warmup_timeout=0 -Pjana:timeout=0 \
0278     -Pplugins=janadot \
0279     "${FULL_TEMP}/${TASKNAME}.edm4hep.root"
0280   if [ -f jana.dot ] ; then mv jana.dot ${LOG_TEMP}/${TASKNAME}.eicrecon.dot ; fi
0281   ls -al ${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root
0282 } 2>&1 | tee ${LOG_TEMP}/${TASKNAME}.eicrecon.log | tail -n1000
0283
0284 # List log files
0285 ls -al ${LOG_TEMP}/${TASKNAME}.*
0286
0287 # Build metadata JSON string for Rucio registration
0288 # Extract software release from eic-info: strip trailing (-default)?-<40hexchars> in one pass
0289 JUG_XL_TAG=$(eic-info 2>/dev/null | grep -oP '(?<=jug_dev: )([\d.]+-(?=stable)|.*?\K)(stable|unstable|nightly|default)')
0290 # Extract metadata from FULL file via podio (all fields except software_release)
0291 PODIO_ARGS=("${FULL_TEMP}/${TASKNAME}.edm4hep.root")
0292 if [[ "$EXTENSION" != "hepmc3.tree.root" ]]; then
0293   PODIO_ARGS+=(--gun)
0294 fi
0295 if [[ "${BASENAME}" == *"BACKGROUNDS"* ]]; then
0296   PODIO_ARGS+=(--no-beam)
0297 fi
0298 PODIO_JSON=$(python $SCRIPT_DIR/parse_podio_metadata.py "${PODIO_ARGS[@]}")
0299
0300 # Only software_release remains outside podio scope
0301 METADATA_JSON_BASE=$(jq -n --arg software_release "${JUG_XL_TAG}" '{software_release: $software_release}')
0302
0303 # Merge podio-extracted fields (geometry_config, data_level, beam/gun params)
0304 METADATA_JSON_FULL=$(jq -n --argjson base "${METADATA_JSON_BASE}" --argjson podio "${PODIO_JSON}" '$base * $podio')
0305 METADATA_JSON_RECO=$(jq -n --argjson base "${METADATA_JSON_FULL}" '$base | .data_level = "reconstruction"')
0306
0307 # Data egress to directory
0308
0309 if [ "${COPYLOG:-false}" == "true" ] ; then
0310   if [ "${USERUCIO:-false}" == "true" ] ; then
0311     TIME_TAG=$(date --iso-8601=second)
0312     TARFILE="${LOG_TEMP}/${TASKNAME}.log.tar.gz"
0313
0314     # Initialize an empty array to hold existing files
0315     FILES_TO_TAR=()
0316
0317     # List of expected files
0318     for FILE in \
0319       "${LOG_TEMP}/${TASKNAME}.npsim.prmon.txt" \
0320       "${LOG_TEMP}/${TASKNAME}.npsim.log" \
0321       "${LOG_TEMP}/${TASKNAME}.eicrecon.prmon.txt" \
0322       "${LOG_TEMP}/${TASKNAME}.eicrecon.log" \
0323       "${LOG_TEMP}/${TASKNAME}.eicrecon.dot" \
0324       "${LOG_TEMP}/${TASKNAME}.hepmcmerger.log"
0325     do
0326       if [ -f "$FILE" ]; then
0327         FILES_TO_TAR+=("$FILE")
0328       fi
0329     done
0330
0331     # Create the tar archive only if there are files to include
0332     if [ ${#FILES_TO_TAR[@]} -gt 0 ]; then
0333       tar -czvf "$TARFILE" "${FILES_TO_TAR[@]}"
0334     else
0335       echo "No log files found to archive."
0336     fi
0337
0338     python $SCRIPT_DIR/register_to_rucio.py \
0339     -f "${LOG_TEMP}/${TASKNAME}.log.tar.gz" \
0340     -d "/${LOG_DIR}/${TASKNAME}.${TIME_TAG}.log.tar.gz" \
0341     -s epic -r ${LOG_RSE:-isLogRSE} --noregister
0342   else
0343     echo "=== DEBUG: Attempting to copy LOG files to xrootd ==="
0344     setup_xrd_auth
0345     echo "Source: ${LOG_TEMP}/${TASKNAME}.*"
0346     echo "Destination: ${XRDWURL}/${XRDWBASE}/${LOG_DIR}"
0347     if [ -n ${XRDWURL} ] ; then
0348       echo "Creating directory: xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${LOG_DIR}"
0349       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${LOG_DIR} || echo "ERROR: Cannot create log directory on xrootd server"
0350     fi
0351     echo "Running: xrdcp --debug 2 --force --recursive ${LOG_TEMP}/${TASKNAME}.* ${XRDWURL}/${XRDWBASE}/${LOG_DIR}"
0352     xrdcp --debug 2 --force --recursive ${LOG_TEMP}/${TASKNAME}.* ${XRDWURL}/${XRDWBASE}/${LOG_DIR} || echo "ERROR: xrdcp failed with exit code $?"
0353     echo "=== DEBUG: LOG copy attempt completed ==="
0354   fi
0355 fi
0356
0357 if [ "${COPYFULL:-false}" == "true" ] ; then
0358   # Validate ROOT file before transfer
0359   echo "=== Validating FULL ROOT file before transfer ==="
0360   python $SCRIPT_DIR/validate_rootfile.py "${FULL_TEMP}/${TASKNAME}.edm4hep.root"
0361   if [ $? -ne 0 ]; then
0362     echo "ERROR: FULL ROOT file validation failed. Skipping transfer."
0363     exit 65
0364   fi
0365   echo "FULL ROOT file validation passed."
0366
0367   if [ "${USERUCIO:-false}" == "true" ] ; then
0368     python $SCRIPT_DIR/register_to_rucio.py -f "${FULL_TEMP}/${TASKNAME}.edm4hep.root" -d "/${FULL_DIR}/${TASKNAME}.edm4hep.root" -s epic -r ${OUT_RSE:-EIC-XRD} --metadata-json "${METADATA_JSON_FULL}" || { echo "ERROR: Rucio registration failed for FULL file."; exit 78; }
0369   else
0370     echo "=== DEBUG: Attempting to copy FULL files to xrootd ==="
0371     setup_xrd_auth
0372     echo "Source: ${FULL_TEMP}/${TASKNAME}.edm4hep.root"
0373     echo "Destination: ${XRDWURL}/${XRDWBASE}/${FULL_DIR}"
0374     if [ -n ${XRDWURL} ] ; then
0375       echo "Creating directory: xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${FULL_DIR}"
0376       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${FULL_DIR} || echo "ERROR: Cannot create simulation directory on xrootd server"
0377     fi
0378     echo "Running: xrdcp --debug 2 --force --recursive ${FULL_TEMP}/${TASKNAME}.edm4hep.root ${XRDWURL}/${XRDWBASE}/${FULL_DIR}"
0379     xrdcp --debug 2 --force --recursive ${FULL_TEMP}/${TASKNAME}.edm4hep.root ${XRDWURL}/${XRDWBASE}/${FULL_DIR} || echo "ERROR: xrdcp failed with exit code $?"
0380     echo "=== DEBUG: FULL copy attempt completed ==="
0381   fi
0382 fi
0383
0384 if [ "${COPYRECO:-false}" == "true" ] ; then
0385   # Validate ROOT file before transfer
0386   echo "=== Validating RECO ROOT file before transfer ==="
0387   python $SCRIPT_DIR/validate_rootfile.py "${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root"
0388   if [ $? -ne 0 ]; then
0389     echo "ERROR: RECO ROOT file validation failed. Skipping transfer."
0390     exit 65
0391   fi
0392   echo "RECO ROOT file validation passed."
0393
0394   if [ "${USERUCIO:-false}" == "true" ] ; then
0395     python $SCRIPT_DIR/register_to_rucio.py -f "${RECO_TEMP}/${TASKNAME}.eicrecon.edm4eic.root" -d "/${RECO_DIR}/${TASKNAME}.eicrecon.edm4eic.root" -s epic -r ${OUT_RSE:-EIC-XRD} --metadata-json "${METADATA_JSON_RECO}" || { echo "ERROR: Rucio registration failed for RECO file."; exit 78; }
0396   else
0397     echo "=== DEBUG: Attempting to copy RECO files to xrootd ==="
0398     setup_xrd_auth
0399     echo "Source: ${RECO_TEMP}/${TASKNAME}*.edm4eic.root"
0400     echo "Destination: ${XRDWURL}/${XRDWBASE}/${RECO_DIR}"
0401     if [ -n ${XRDWURL} ] ; then
0402       echo "Creating directory: xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${RECO_DIR}"
0403       xrdfs ${XRDWURL} mkdir -p ${XRDWBASE}/${RECO_DIR} || echo "ERROR: Cannot create reconstruction directory on xrootd server"
0404     fi
0405     echo "Running: xrdcp --debug 2 --force --recursive ${RECO_TEMP}/${TASKNAME}*.edm4eic.root ${XRDWURL}/${XRDWBASE}/${RECO_DIR}"
0406     xrdcp --debug 2 --force --recursive ${RECO_TEMP}/${TASKNAME}*.edm4eic.root ${XRDWURL}/${XRDWBASE}/${RECO_DIR} || echo "ERROR: xrdcp failed with exit code $?"
0407     echo "=== DEBUG: RECO copy attempt completed ==="
0408   fi
0409 fi
0410
0411 # closeout
0412 date
0413 find ${TMPDIR}
0414 du -sh ${TMPDIR}