job_submission_condor/scripts/submit_csv.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # check arguments
0007 if [ $# -lt 2 ] ; then
0008   echo "Usage: "
0009   echo "  $0 <template> <type> <file or url> [target hours = 2]"
0010   exit
0011 fi
0012
0013 # Project configuration
0014 BASEURL="https://eicweb.phy.anl.gov/api/v4/projects/491/jobs/artifacts/${DETECTOR_VERSION:-main}/raw/results/datasets/timings/"
0015 BASEJOB="?job=collect"
0016 BKGURL="https://eicweb.phy.anl.gov/EIC/campaigns/datasets/-/raw/main/config_data"
0017
0018 # Parse arguments
0019 # - condor template
0020 TEMPLATE=${1}
0021 shift
0022 # - type of simulation
0023 TYPE=${1}
0024 shift
0025 # - input file
0026 FILE=${1}
0027 shift
0028 # - target hours
0029 TARGET=${1:-2}
0030 shift
0031
0032 # process csv file into jobs
0033 if [ -n "${CSV_FILE:-}" ]; then
0034   # allow to set custom csv file for job instead of fetching from web archive
0035   CSV_FILE=$(realpath -e ${CSV_FILE})
0036 else
0037   CSV_FILE=$($(dirname $0)/csv_to_chunks.sh ${FILE} ${TARGET})
0038 fi
0039
0040 # create command line
0041 EXECUTABLE="$(dirname $0)/run.sh"
0042 ARGUMENTS="EVGEN/\$(file) \$(ext) \$(nevents) \$(ichunk)"
0043
0044 # Set background environment variables
0045 if [ -n "${BG_FILES:-}" ]; then
0046   curl -L -O ${BKGURL}/${BG_FILES}
0047 fi
0048
0049 # construct environment file
0050 ENVIRONMENT=environment-$(basename ${CSV_FILE} .csv).sh
0051
0052 # extract certificate name
0053 X509_USER_PROXY_BASE=$(basename ${X509_USER_PROXY:-})
0054
0055 sed "
0056   s|%COPYRECO%|${COPYRECO:-}|g;
0057   s|%COPYFULL%|${COPYFULL:-}|g;
0058   s|%COPYLOG%|${COPYLOG:-}|g;
0059   s|%USERUCIO%|${USERUCIO:-}|g;
0060   s|%X509_USER_PROXY%|${X509_USER_PROXY_BASE:-}|g;
0061   s|%TAG_PREFIX%|${TAG_PREFIX:-}|g;
0062   s|%TAG_SUFFIX%|${TAG_SUFFIX:-}|g;
0063   s|%DETECTOR_VERSION%|${DETECTOR_VERSION}|g;
0064   s|%DETECTOR_CONFIG%|${DETECTOR_CONFIG}|g;
0065   s|%EBEAM%|${EBEAM}|g;
0066   s|%PBEAM%|${PBEAM}|g;
0067   s|%SIGNAL_FREQ%|${SIGNAL_FREQ:-}|g;
0068   s|%SIGNAL_STATUS%|${SIGNAL_STATUS:-}|g;
0069   s|%BG_FILES%|${BG_FILES:-}|g;
0070 " templates/${TEMPLATE}.sh.in > ${ENVIRONMENT}
0071
0072 # construct requirements
0073 REQUIREMENTS=""
0074
0075 # construct input files
0076 INPUT_FILES=${ENVIRONMENT},${X509_USER_PROXY}
0077 INPUT_FILES="${INPUT_FILES}${BG_FILES:+,${BG_FILES}}"
0078
0079 # construct submission file
0080 SUBMIT_FILE=$(basename ${CSV_FILE} .csv).submit
0081 sed "
0082   s|%EXECUTABLE%|${EXECUTABLE}|g;
0083   s|%ARGUMENTS%|${ARGUMENTS}|g;
0084   s|%JUG_XL_TAG%|${JUG_XL_TAG:-nightly}|g;
0085   s|%DETECTOR_VERSION%|${DETECTOR_VERSION}|g;
0086   s|%DETECTOR_CONFIG%|${DETECTOR_CONFIG}|g;
0087   s|%INPUT_FILES%|${INPUT_FILES}|g;
0088   s|%REQUIREMENTS%|${REQUIREMENTS}|g;
0089   s|%CSV_FILE%|${CSV_FILE}|g;
0090 " templates/${TEMPLATE}.submit.in > ${SUBMIT_FILE}
0091
0092 # submit job
0093 condor_submit -verbose -file ${SUBMIT_FILE}
0094
0095 # create log dir
0096 if [ $? -eq 0 ] ; then
0097   for i in `condor_q --batch | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do
0098     mkdir -p LOG/CONDOR/osg_$i/
0099   done
0100 fi