simulation_campaign_datasets/scripts/split_in_chunks.sh

0001 #!/bin/bash
0002 set -Euo pipefail
0003 trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
0004 IFS=$'\n\t'
0005
0006 file=${1?Specify filename}
0007 nevents=${2?Specify nevents}
0008
0009 # break every 10k events
0010 zeroes="0000"
0011 events_per_chunk=1${zeroes}
0012 pattern="/^E .*${zeroes} /"
0013
0014 # handle gzip inputs
0015 if [[ "${file}" =~ gz$ ]] ; then
0016   CAT=(zcat)
0017   GZIP=(gzip)
0018 else
0019   CAT=(cat)
0020   GZIP=(cat)
0021 fi
0022
0023 # get header and footer first
0024 header=`${CAT[@]} ${file} | sed -n '/^E/q;p'`
0025 footer="HepMC::Asciiv3-END_EVENT_LISTING"
0026
0027 # determine padding
0028 pad=%0$(echo "scale=0; l(${nevents}/${events_per_chunk})/l(10)+1" | bc -l)d
0029 echo "${nevents} events in chunks of ${events_per_chunk} requires ${pad} padding"
0030
0031 # split
0032 suffix=".csplit."
0033 prefix=${file}${suffix}
0034 echo "Splitting ${file} on pattern ${pattern}"
0035 ${CAT[@]} ${file} | csplit --elide-empty-files --quiet --prefix=${prefix} --suffix-format=${pad} - "%^E%" "${pattern}" "{*}"
0036
0037 # add header and footer, compress
0038 echo "Adding headers and footers again (and compressing)"
0039 for chunkfile in ${prefix}* ; do
0040   chunk=${chunkfile/*${suffix}/}
0041   output=${file/.hepmc/_${chunk}.hepmc}
0042   {
0043     echo "${header}"
0044     cat ${chunkfile}
0045     echo "${footer}"
0046   } | ${GZIP} > ${output}
0047   echo "${output}"
0048   rm ${chunkfile}
0049   # FIXME double footer on last segment is harmless
0050 done