source: utils/tools/REBUILD_NEMO/rebuild_nemo @ 10727

Last change on this file since 10727 was 10338, checked in by mathiot, 2 years ago

Add deflation/chunking to rebuild_nemo tool. Fix #2165

  • Property svn:executable set to *
File size: 4.5 KB
Line 
1#!/bin/ksh
2
3# CALLS: rebuild_nemo.exe
4
5#set -ax
6usage ()
7{
8   echo
9   echo "  NEMO Rebuild"
10   echo "  ************"
11   echo
12   echo "  usage: ${0##*/} [-l -p -s -m -n -r -d -x -y -z -t -c] filebase ndomain [rebuild dimensions]"
13   echo
14   echo "  flags:    -l arch            submit to compute node"
15   echo "            -p num             use num threads"
16   echo "            -s num             split 4D vars into time slice of size num"
17   echo "            -m                 force masking of global arrays (zero if no mdi)"
18   echo "            -n namelist        full path to namelist file to be created (otherwise default nam_rebuild+_process_id is used)"
19   echo "            -r memory          Memory to request on compute node including units (Default = 10Gb)"
20   echo ""
21   echo "      key_netcdf4 only "
22   echo "            -d deflate_level     deflate level for output files"
23   echo "            -x chunksize along x " 
24   echo "            -y chunksize along y " 
25   echo "            -z chunksize along z " 
26   echo "            -t chunksize along t " 
27   echo "            -c total size of the chunk cache "
28   echo
29   exit 1
30}
31
32while getopts l:p:s:n:r:d:x:y:z:t:c:m opt
33do
34  case ${opt} in
35      l) 
36         BATCH="yes"
37         ARCH=${OPTARG}
38      ;;
39      p)
40         OMP_NUM_THREADS=${OPTARG}
41      ;;
42      s)
43         NSLICESIZE=${OPTARG}
44      ;;
45      m) 
46         NMASK="TRUE"
47         echo ""
48         echo "output is mask using netcdf missing value (_Fillvalue attribute) or 0 if missing value not in the netcdf."
49         echo ""
50      ;;
51      d)
52         DEFLATE=${OPTARG}
53      ;;
54      n)
55         nam_rebuild=${OPTARG}
56      ;;
57      r)
58         MEMORY=${OPTARG}
59      ;;
60      x)
61         NXCHUNK=${OPTARG}
62      ;;
63      y)
64         NYCHUNK=${OPTARG}
65      ;;
66      z)
67         NZCHUNK=${OPTARG}
68      ;;
69      t)
70         NTCHUNK=${OPTARG}
71      ;;
72      c)
73         CHUNKSIZE=${OPTARG}
74      ;;
75  esac
76done
77shift $(expr ${OPTIND} - 1)
78
79if [[ $# -lt 2 ]] ; then
80   usage
81fi
82
83script_dir=$(dirname $0)
84
85file=$1
86ndomain=$2
87DIM1=$3
88DIM2=$4
89export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
90nam_rebuild=${nam_rebuild:-nam_rebuild_$$}
91MEMORY=${MEMORY:-10Gb}
92
93#Find out the maximum number of files that can be opened and increase if necessary)
94nopen=$(ulimit -n)
95if [[ $ndomain -gt $nopen ]] ; then
96  nopen=$((ndomain+4))   # +2 failed !!!
97fi
98
99if [[ -n ${DIM1} && -n ${DIM2} ]] ; then
100   dim_str=" dims '${DIM1}','${DIM2}'"
101   dims="dims='${DIM1}','${DIM2}'"
102fi
103
104echo "file ${file},  num_domains ${ndomain}, num_threads ${OMP_NUM_THREADS}${dim_str}"
105
106cat > $nam_rebuild << EOC
107&nam_rebuild
108filebase='${file}'
109ndomain=${ndomain}
110EOC
111if [[ -n ${dims} ]] ; then
112   echo ${dims} >> $nam_rebuild
113fi
114if [[ -n ${NCSLICESIZE} ]] ; then
115   echo " nslicesize=${NCSLICESIZE}" >> $nam_rebuild
116fi
117if [[ -n ${NMASK} ]] ; then
118   echo " l_maskout=.true." >> $nam_rebuild
119fi
120if [[ -n ${DEFLATE} ]] ; then
121   echo " deflate_level=${DEFLATE}" >> $nam_rebuild
122fi
123if [[ -n ${NXCHUNK} ]] ; then
124   echo " nc4_xchunk=${NXCHUNK}" >> $nam_rebuild
125fi
126if [[ -n ${NYCHUNK} ]] ; then
127   echo " nc4_ychunk=${NYCHUNK}" >> $nam_rebuild
128fi
129if [[ -n ${NZCHUNK} ]] ; then
130   echo " nc4_zchunk=${NZCHUNK}" >> $nam_rebuild
131fi
132if [[ -n ${NTCHUNK} ]] ; then
133   echo " nc4_tchunk=${NTCHUNK}" >> $nam_rebuild
134fi
135if [[ -n ${CHUNKSIZE} ]] ; then
136   echo " fchunksize=${CHUNKSIZE}" >> $nam_rebuild
137fi
138
139
140
141echo "/" >> $nam_rebuild
142
143if [[ ${BATCH} == "yes" ]] ; then
144
145   template_dir=${script_dir}/BATCH_TEMPLATES/
146   param_file=${template_dir}/param_${ARCH}
147   if [ ! -f $param_file ]; then
148      echo ''
149      echo "E R R O R: $param_file is missing, stop 42"
150      echo ''
151      echo "check your arch name or add one $param_file file in BATCH_TEMPLATES"
152      echo ''
153      exit 42
154   fi
155   . $param_file
156
157   batch_file=rebuild_nemo_batch_${ARCH}
158   if [ ! -f ${template_dir}/${batch_file} ]; then
159      echo ''
160      echo "E R R O R: $batch_file is missing, stop 42"
161      echo ''
162      echo "check your arch name or add one $batch_file file in BATCH_TEMPLATES"
163      echo ''
164      exit 42
165   fi
166
167   #Create a modified local copy of the batch submission file
168   #The process ID is appended to the end of the file name so it is unique
169   cat ${template_dir}/${batch_file} | sed -e"s/NTHREADS/${OMP_NUM_THREADS}/" \
170      -e"s/MEMORY/${MEMORY}/" \
171      -e"s:INDIR:${script_dir}:" \
172      -e"s/NOPEN/${nopen}/" \
173      -e"s/NAMELIST/${nam_rebuild}/" > ${batch_file}_$$.sh
174   
175   #Submit the job
176   echo "Submitting job to compute node"
177   $BATCH_CMD ${batch_file}_$$.sh
178
179else
180   ulimit -n $nopen
181   ${script_dir}/rebuild_nemo.exe $nam_rebuild
182fi
183
Note: See TracBrowser for help on using the repository browser.