job_submission_condor/scripts/csv_to_chunks.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 # check arguments
0007 if [ $# -lt 2 ] ; then
0008   echo "Usage: "
0009   echo "  $0 <file or url> [target hours = 2]"
0010   exit
0011 fi
0012
0013 # Parse arguments
0014 # - input file
0015 FILE=${1}
0016 shift
0017 # - target hours
0018 TARGET=${1:-2}
0019 shift
0020
0021 # Parse input URI
0022 if [ -f "${FILE}" ] ; then
0023   echo "Using local file as input" 1>&2
0024   INPUT=(cat ${FILE})
0025 else
0026   echo "Using ${BASEURL}${FILE}${BASEJOB} as input" 1>&2
0027   INPUT=(curl ${BASEURL}${FILE}${BASEJOB})
0028 fi
0029
0030 # Output file
0031 output=$(basename ${FILE} .csv)-$(date --iso-8601=minutes).csv
0032
0033 # Loop over input
0034 "${INPUT[@]}" | grep -v ^curl | while IFS="," read file ext ntotal dt0 dt1 duf0 duf1 dur0 dur1 ; do
0035   if [[ "${file}" =~ csv$ ]] ; then
0036     ${0} ${TEMPLATE} ${TYPE} ${file} ${TARGET}
0037   else
0038     nevents=$(echo "scale=0; n=(3600*$TARGET-$dt0)/$dt1; if (n>$ntotal) print($ntotal) else print(n)" | bc -l)
0039     if [ -n "${MAX_EVENTS_PER_CHUNK:-}" ] && { [ "${nevents}" -le 0 ] || [ "${nevents}" -gt "${MAX_EVENTS_PER_CHUNK}" ]; }; then
0040       nevents=${MAX_EVENTS_PER_CHUNK}
0041     fi
0042     nchunks=$(echo "scale=0; n=$ntotal/$nevents+1; if (n==0) print(1) else print(n)" | bc -l)
0043     nevents=$(echo "scale=0; $ntotal/$nchunks" | bc -l)
0044     actualt=$(echo "scale=2; ($dt0+$nevents*$dt1)/3600" | bc -l)
0045     for ichunk in `seq 0 $((nchunks-1))` ; do
0046       echo "${file},${ext},${nevents},$(printf '%04d' $ichunk)"
0047     done
0048   fi
0049 done > ${output}
0050 echo "${output}"