IPSLCM6/IPSL-CM6A-LR: batch_merge_EXT.sh

#!/bin/bash
#MSUB -r merge_EXT
#MSUB -n 16
#MSUB -T 86400
#MSUB -q skylake
#MSUB -A devcmip6
#MSUB -o merge_EXT.o
#MSUB -e merge_EXT.e
#MSUB -m work

module load glost
module load nco

SIMULATION=/ccc/work/cont003/gencmip6/p86denv/IGCM_OUT/IPSLCM6/PROD/historical/CM61-LR-histEXT-03.1890/CMIP6/

if [ $(basename $SIMULATION) != "CMIP6" ]; then
    echo ":: ERROR   :: Basename of your SIMULATION should be 'CMIP6'"
    exit 1
fi

echo ":: INFO    :: Copy data from $SIMULATION to ${SIMULATION/CMIP6/CMIP6_merge} using hard links"
rm -fr ${SIMULATION/CMIP6/CMIP6_merge}
cp -al $SIMULATION ${SIMULATION/CMIP6/CMIP6_merge}
SIMULATION=${SIMULATION/CMIP6/CMIP6_merge}

rm -f $SIMULATION/TaskFile.txt

echo ":: INFO    :: Find all filename patterns"
find $SIMULATION -type f ! -name "*_fx_*" ! -name "*_?fx_*" | xargs -i basename {}| cut -d _ -f 1-6 | sort -u > $SIMULATION/file_patterns.txt
total=$(cat $SIMULATION/file_patterns.txt | wc -l)
echo ":: INFO    :: ${total} patterns founds"

echo ":: INFO    :: Create task file"
for file_pattern in $(cat $SIMULATION/file_patterns.txt); do
    find $SIMULATION -type f -name "${file_pattern}*" > $SIMULATION/files.txt
    dir=$(cat $SIMULATION/files.txt | xargs -i dirname {} | sort -u)
    first=$(cat $SIMULATION/files.txt |  xargs -i basename {} | cut -d "_" -f 7 | sed 's|-.*||g' | sort | head -1)
    last=$(cat $SIMULATION/files.txt |  xargs -i basename {} | cut -d "_" -f 7 | sed 's|-.*||g' | sort | tail -1)
    echo "ncrcat \$(ls $dir/${file_pattern}_* | sort) $dir/${file_pattern}_$first-$last.nc" >> $SIMULATION/TaskFile.txt
    cat $SIMULATION/files.txt >> $SIMULATION/to_remove.txt
done

ccc_mprun glost_launch $SIMULATION/TaskFile.txt

echo ":: INFO    :: Remove netCDF files."
cat $SIMULATION/to_remove.txt | xargs -i rm -fr {}

echo ":: INFO    :: Remove temporary files."
rm -fr $SIMULATION/files.txt $SIMULATION/file_patterns.txt $SIMULATION/to_remove.txt

echo ":: INFO    :: Set ACLs on $SIMULATION"
setfacl -Rm u:levavasg:rwX,u:p86denv:rwX $SIMULATION
setfacl -Rdm u:levavasg:rwX,u:p86denv:rwX $SIMULATION