IPSLCM6/IPSL-CM6A-LR: batch_merge_EXT.sh

File batch_merge_EXT.sh, 2.0 KB (added by glipsl, 2 years ago)
Line 
1#!/bin/bash
2#MSUB -r merge_EXT
3#MSUB -n 16
4#MSUB -T 86400
5#MSUB -q skylake
6#MSUB -A devcmip6
7#MSUB -o merge_EXT.o
8#MSUB -e merge_EXT.e
9#MSUB -m work
10
11module load glost
12module load nco
13
14SIMULATION=/ccc/work/cont003/gencmip6/p86denv/IGCM_OUT/IPSLCM6/PROD/historical/CM61-LR-histEXT-03.1890/CMIP6/
15
16if [ $(basename $SIMULATION) != "CMIP6" ]; then
17    echo ":: ERROR   :: Basename of your SIMULATION should be 'CMIP6'"
18    exit 1
19fi
20
21echo ":: INFO    :: Copy data from $SIMULATION to ${SIMULATION/CMIP6/CMIP6_merge} using hard links"
22rm -fr ${SIMULATION/CMIP6/CMIP6_merge}
23cp -al $SIMULATION ${SIMULATION/CMIP6/CMIP6_merge}
24SIMULATION=${SIMULATION/CMIP6/CMIP6_merge}
25
26rm -f $SIMULATION/TaskFile.txt
27
28echo ":: INFO    :: Find all filename patterns"
29find $SIMULATION -type f ! -name "*_fx_*" ! -name "*_?fx_*" | xargs -i basename {}| cut -d _ -f 1-6 | sort -u > $SIMULATION/file_patterns.txt
30total=$(cat $SIMULATION/file_patterns.txt | wc -l)
31echo ":: INFO    :: ${total} patterns founds"
32
33echo ":: INFO    :: Create task file"
34for file_pattern in $(cat $SIMULATION/file_patterns.txt); do
35    find $SIMULATION -type f -name "${file_pattern}*" > $SIMULATION/files.txt
36    dir=$(cat $SIMULATION/files.txt | xargs -i dirname {} | sort -u)
37    first=$(cat $SIMULATION/files.txt |  xargs -i basename {} | cut -d "_" -f 7 | sed 's|-.*||g' | sort | head -1)
38    last=$(cat $SIMULATION/files.txt |  xargs -i basename {} | cut -d "_" -f 7 | sed 's|-.*||g' | sort | tail -1)
39    echo "ncrcat \$(ls $dir/${file_pattern}_* | sort) $dir/${file_pattern}_$first-$last.nc" >> $SIMULATION/TaskFile.txt
40    cat $SIMULATION/files.txt >> $SIMULATION/to_remove.txt
41done
42
43ccc_mprun glost_launch $SIMULATION/TaskFile.txt
44
45echo ":: INFO    :: Remove netCDF files."
46cat $SIMULATION/to_remove.txt | xargs -i rm -fr {}
47
48echo ":: INFO    :: Remove temporary files."
49rm -fr $SIMULATION/files.txt $SIMULATION/file_patterns.txt $SIMULATION/to_remove.txt
50
51echo ":: INFO    :: Set ACLs on $SIMULATION"
52setfacl -Rm u:levavasg:rwX,u:p86denv:rwX $SIMULATION
53setfacl -Rdm u:levavasg:rwX,u:p86denv:rwX $SIMULATION