source: TOOLS/CMIP6_FORCING/OZONE/tools.bash @ 5567

Last change on this file since 5567 was 4109, checked in by cugnet, 6 years ago

Available datasets list on ciclad extended.
Scripts slightly modified to accomodate more ways of cutting a time serie into several time slices.

File size: 13.7 KB
Line 
1function parse_dir {
2# Syntax:   parse_dir <dir> <dir_struct> [<patch1 ... patchN>]
3#       $1:     directory to parse
4#       $2:     directory structure
5#      [$3...]: <dset>:<var>=<val> patches
6# Purpose: parse directory <dir> with structure <dir_struct> and export all variables.
7# If $dataset = $dset, the value of <var> obtain with parsing is replaced with <val>.
8  local d s
9  d=$1; shift; s=$1; shift
10  while :; do
11    $(eval echo "export ${s%%/*}=${d%%/*}")
12    s=${s#*/}; d=${d#*/}
13    if [ "$s" = "${s#*/}" ]; then break; fi
14  done
15  while [ $# -ge 1 ]; do if [ $dataset = ${1%:*} ]; then eval export ${1#*:}; fi; shift; done
16  unset d s
17}
18#==================================================================================
19function replace_1 {
20  # $1: string  $2: <patch>=<replacement string>
21  local out=$1
22  while [ "${out%${2%=*}*}" != "$out" ]; do out=${out%${2%=*}*}"${2#*=}"${out##*${2#=*}*}; done
23  echo $out; unset out; return 0
24}
25#==================================================================================
26function replace {
27  # same as "replace_1", but with several <patch>=<replacement string> arguments.
28  local out=$1; shift
29  while [ $# -ne 0 ]; do out=$(replace_1 $out "$1"); shift; done
30  echo $out; unset out; return 0
31}
32#==================================================================================
33function addc {
34  local s="$1"; while [ ${#s} -lt $2 ]; do s="_$s"; done ; echo "$s"; unset s
35}
36#==================================================================================
37function progress_bar {
38# $1: iteration  $2: iterations number  $3: length of the bar
39  local n bar
40  bar=`echo "scale=1 ; (100*$1)/$2" | bc`
41  bar="`addc "$bar" 5`% ["
42  n=`echo "scale=0 ; ($3*$1)/$2" | bc`
43  if [ $n -ne 0 ]; then bar="$bar"`eval "printf '='%.0s {1..$n}"`; fi
44  n=$(($3-$n))
45  if [ $n -ne 0 ]; then bar="$bar"`eval "printf '.'%.0s {1..$n}"`']\r'; fi
46  echo -ne $bar; if [ $2 -eq $1 ]; then echo; fi
47  unset n bar
48}
49#==================================================================================
50function extract {
51# Input arguments:
52#    $1: v_in,v_ou     (input/output variable names)
53#    $2: f_in,f_ou     (input/output files names)
54#    $3: Ymin,Ymax,dY  (min/max available years, years/file number)
55#    $4: nmrg          (number of additional months before and after)
56#
57# Purpose:
58#  * Extract year $Y for variable $v_in from files $f_in containing data from
59#    $Ymin to $Ymax, where each input file contains $dY years
60#  * Outputs are stored in file $f_ou, and variable renamed $v_ou.
61#  * Optionally, add before/after the 12 months the $nmrg previous/next months.
62#    - Y=Ymin (missing Ymin-1) => $nmrg last  months of year $Y used instead.
63#    - Y=Ymax (missing Ymax+1) => $nmrg first months of year $Y used instead.
64#
65# Remark:  First record of first file has to be january $Ymin.
66#
67  local v_in v_ou d_in f_ou Ymin Ymax Yb Ye Y dY nbm nmrg it fin fou fm ff fp ib ie
68  v_in="${1%%,*}"; v_ou="${1#*,}"; shift
69  f_in="${1%%,*}"; f_ou="${1#*,}"; shift
70  Ymin="${1%%,*}"; Ymax="${1#*,}"; dY="${Ymax#*,}"; Ymax="${Ymax%%,*}"; shift
71  Y=$1; shift; if [ $# -eq 1 ]; then nmrg=$1; else nmrg=0; fi
72  if [ $dY -eq $((Ymax-Ymin+1)) ]; then dY=0; fi    #--- SINGLE FILE
73  ff=actu_${v_in}_${Y}.nc; fm=""; fp=""
74  #--- INITIAL AND FINAL YEAR OF FILE CONTAINING CURRENT YEAR
75  if [ $dY -eq 0 ]; then
76    Yb=$Ymin; Ye=$Ymax
77  else
78    Yb=$((Ymin+dY*((Y-Ymin)/dY))); Ye=$((Yb+dY-1))
79    Ye=$(($Ye>$Ymax?$Ymax:$Ye))
80  fi
81  nbm=$((12*(Ye-Yb+1)-1))                           #--- MONTHS NUMBER OF CURRENT FILE
82  it=$((12*(Y-Yb)))                                 #--- JANUARY INDEX IN CURRENT FILE
83  fin=`var=$v_in; Yb=$Yb; Ye=$Ye; eval echo $f_in`  #--- CURRENT FILE NAME
84  fou=`var=$v_ou; Y=$Y; eval echo $f_ou`            #--- OUTPUT  FILE NAME
85  ib=$(($it<=nmrg?0:$((it-nmrg))))                  #--- INDEX OF FIRST USED MONTH
86  ie=$(($((it+11+mrg))>=$nbm?$nbm:$((it+11+nmrg)))) #--- INDEX OF LAST USED MONTHF
87  rm -f $ff ; rm -f $ff ; ncks -d time,$ib,$ie $fin $ff
88  #--- MISSING PREVIOUS YEAR LAST  MONTHS => CURRENT YEAR INSTEAD
89  if [ $it           -lt $nmrg ]; then fm="prev_${v_in}_${Y}.nc"; rm -f $fm
90    if [[ $Y -eq $Ymin || $dY -eq 0  ]]; then ncks -d time,$((it-nmrg+12)),11 $fin $fm
91    else Yb=$((dY*((Y-1)/dY))); Ye=$((Yb+dY-1)); Ye=$(($Ye>$Ymax?$Ymax:$Ye))
92      ncks -d time,$((12*(Ye-Yb+1)-nmrg+it)),$((12*(Ye-Yb+1)-1)) `var=$v_in; Yb=$Yb; Ye=$Ye; eval echo $f_in` $fm
93    fi
94  fi
95  #--- MISSING FOLLOWING YEAR FIRST MONTHS => CURRENT YEAR INSTEAD
96  if [ $((it+nmrg+11)) -gt $nbm ]; then fp="next_${v_in}_${Y}.nc"; rm -f $fp
97    if [[ $Y -eq $Ymax || $dY -eq 0  ]]; then ncks -d time,$((nbm-11)),$((it+nmrg-1)) $fin $fp
98    else Yb=$((dY*((Y+1)/dY))); Ye=$((Yb+dY-1)); Ye=$(($Ye>$Ymax?$Ymax:$Ye))
99      ncks -d time,0,$((nmrg-1)) `var=$v_in; Yb=$Yb; Ye=$Ye; eval echo $f_in` $fp
100    fi
101  fi
102  if [ ! -d ${fou%/*} ]; then mkdir -p ${fou%/*}; fi
103  if [[ "$fm" != "" || "$fp" != "" ]]; then
104    rm -f $fou ; ncrcat $fm $ff $fp $fou ; rm -f $fm $ff $fp
105  else
106    mv $ff $fou
107  fi
108  if [ "$v_in" != "$v_ou" ]; then
109    ncrename -v $v_in,$v_ou $fou > /dev/null 2>&1
110  fi
111}
112#==================================================================================
113function make_clim {
114# Input arguments:
115#    $1: variable name   $2: input files name
116#    $3: beg-end years   $4: index for january (optional, starting from 0)
117#
118# Purpose:
119#    Create a 12 months climatology for a variable using 1-year monthly files.
120#
121# Remark:
122#    Files with additional months are usable if the index for january is given (default: 0)
123#
124  local f_ou f0 Yb Ye ib ie M MM
125  Yb="${3%%-*}"; Ye="${3#*-}"
126  if [ $# -eq 4 ]; then ib=$4; else ib=0; fi; ie=$((ib+11))  #=== INDEX $ib FOR JANUARY
127  for M in $(eval echo {$ib..$ie}); do
128    MM=$((M+1-ib)); if [ $MM -le 9 ]; then MM=0$MM; fi
129    for Y in `eval echo {${Yb}..${Ye}}`; do
130      f_ou=$(var=$1; Y=$Y; eval echo $2)
131      rm -f ${1}_$Y$MM.nc ; ncks -d time,$M $(var=$1; Y=$Y; eval echo $f_ou) ${1}_$Y$MM.nc
132    done
133    rm -f ${1}_$MM.nc ; ncra ${1}_????$MM.nc ${1}_$MM.nc ; rm -f ${1}_????$MM.nc
134    progress_bar $((M+1-ib)) $((ie-ib+1)) 50
135  done
136  f_ou=$(var=$1; Y=${Yb}_${Ye}_clim; eval echo $2)
137  if [ ! -p ${f_ou%/*} ]; then mkdir -p ${f_ou%/*}; fi
138  rm -f $f_ou ; ncrcat ${1}_??.nc $f_ou ; rm -f ${1}_??.nc
139  f0=`var=$1; Y='YYYY'; eval echo $2`; f0=$(var=$1; Y='YYYY'; eval echo $f0); f0=${f0##*/}
140  ncatted -a comment,global,a,c,"\nClimatology from $Yb to $Ye using file(s) $f0" $f_ou
141}
142#==================================================================================
143function str2ascii() {
144  LC_CTYPE=C printf '%d' "'$1"
145}
146
147#==================================================================================
148function deblank {
149#--- REMOVE UNEXPECTED CHARACTERS (IN PARTICULAR THOSE WEIRD ASCII(0) CHARS)
150  local a
151  a=$1; while [ `str2ascii ${a:0:1}` -lt 32 ]; do a=${a:1:${#a}}; done; echo $a
152}
153
154#==================================================================================
155function dim_nam {
156  #--- GET DIMENSION $2 (X/Y/Z/T) NAME FOR FILE $1
157  #--- SPECIAL CASE: RETURN 'Seasons' IF "$2" = "S".
158  local a v Units
159  case $2 in
160    X) Units='degrees_east  degree_east  degree_e degrees_e "degrees east"  "degree east"  degreee degreese' ;;
161    Y) Units='degrees_north degree_north degree_n degrees_n "degrees north" "degree north" degreen degreesn' ;;
162    Z) Units='pa pascal pascals hpa millibar millibars mbar mbars mb bar bars atm atms atmosphere atmospheres' ;;
163    T) Units=' since' ;;
164    S) echo Seasons; return 0 ;;
165    *) echo "error in dim_nam: unrecognized axis $2"; return 1 ;;
166  esac
167  for u in $Units; do
168    case $2 in
169      X|Y|Z) a=`ncdump -h $1 | grep -i ':units = "'"$u"'"'` ;;
170      T)     a=`ncdump -h $1 | grep -i $u | grep ':units = "'` ;;
171    esac
172    if [ "$a" != "" ]; then
173      if [ "$2" = "Z" ]; then v=${a%%:*}; v=`deblank $v`
174        if [ "`ncdump -h $1 | grep -i "$v($v)"`" != "" ]; then break; fi
175      else break
176      fi
177    fi
178  done
179  if [[ "$a" = "" && "$2" = "Z" ]]; then a=`ncdump -h $1 | grep -i ':positive = "'`; fi
180  if [ "$a" = "" ]; then echo "error in dim_nam: unable to recognize axis $2"; return 1; fi
181  echo ${a%%:*}; return 0
182}
183
184#==================================================================================
185function test_var {
186# Input arguments:
187#    $1: file name  $2: variable name
188  ncdump -h $1 | grep "double $2" > /dev/null; if [ $? -eq 0 ]; then return 0; fi
189  ncdump -h $1 | grep "float  $2" > /dev/null; return $?
190}
191#==================================================================================
192function get_boundsvar {
193# Input arguments:
194#    $1: file name  $2: variable name
195  local a=$(ncdump -h $1 | grep -P "\t$2:bounds")
196  if [ "$a" != "${a#*\"}" ]; then a=${a#*\"}; echo ${a%\"*}; else echo ""; fi
197}
198#==================================================================================
199function rename_bounds {
200# Input arguments:
201#    $1: file name  $2: variable name  $3: new variable name
202#    returns: $old_var,$old_boundvar,$new_boundvar
203  local vin=$(get_boundsvar $1 $2) beg end vou
204  if [ "$vin" = "" ]; then return 1; fi
205  beg=${vin##*$2}; if [ "$beg" = "$vin" ]; then beg=$vin; fi
206  beg=${vin%%$2*}; if [ "$end" = "$vin" ]; then end=$vin; fi
207  if [[  "$beg" = ""  &&  "$end" = ""  ]]; then return 1; fi
208  vou=$beg$3$end
209  echo "$2,$vin,$vou"
210}
211#==================================================================================
212function rename_dims {
213# Input arguments:
214#    $1: file name  $2...: <axis>:<target_name> (axis=X,Y,Z or T)
215#
216# Purpose:
217#    Rename dimensions, dimensional variables.
218#    In case of boundary grid variables with bounds_<variable> names are available,
219#    they are renamed together with the corresponding "bounds" attributes.
220  local vio f=$1 att="" ren="" nam_in nam_ou
221  shift
222  while [ $# -ge 1 ]; do
223    nam_ou=${1##*:}; nam_in=$(dim_nam $f ${1%:*}); shift
224    if [ "$nam_in" != "$nam_ou" ]; then
225      ren="$ren -d $nam_in,$nam_ou -v $nam_in,$nam_ou"
226      #--- CHECK WETHER "bounds" ATTRIBUTE IS THERE
227      vio="$(rename_bounds $f $nam_in $nam_ou)"
228      if [ "$vio" != "" ]; then
229        att="$att -a bounds,$nam_ou,m,c,${vio##*,}"; ren="$ren -v ${vio#*,}"
230      fi
231    fi
232  done
233  if [ "$ren" != "" ]; then ncrename $ren $f 2>&1 > /dev/null ; fi
234  if [ "$att" != "" ]; then ncatted  $att $f 2>&1 > /dev/null ; fi
235}
236#==================================================================================
237function zonal_mean {
238# Input arguments:
239#    $1: input file  $2: output file
240  local tmp=tmp.$$.nc
241  if [ ! -d ${2%/*} ]; then mkdir -p ${2%/*}; fi
242  ncwa       -a longitude $1 $tmp ; mv $tmp $2    #--- Zonal mean
243  ncks -O -x -v longitude $2 $tmp ; mv $tmp $2    #--- Remove longitude
244  v="bounds_longitude"; test_var $2 $v            #--- Remove bounds vars
245  if [ $? -eq 0 ]; then ncks -O -x -v $v $2 $tmp ; mv $tmp $2; fi
246}
247#==================================================================================
248function get_date {
249# Call get_date1 one time each input file.
250# $1: varname    $2: datatype    $3: dirin    $4: <dir_in> patch    $5...$N: <files_in>
251# [$N+1...]: <dset>:<var>=<val> to overwrite value of variables evaluated from parsing.
252  local v_in dtyp d_in dstr f_in Yb Ye St a f first date dates files_in=""
253  v_in=$1; shift; dtyp=$1; shift; d_in=$1; shift; dstr=$1; shift; f0=$1
254  while [ "$1" = "${1##*=}" ]; do files_in="$files_in $1"; shift; done
255  for f_in in $files_in; do date=$(get_date_1 $v_in $dtyp $d_in $dstr $f_in $*)
256    if [ $f_in = $f0 ]; then dates=$date; else dates="$dates $date"; fi
257  done
258  echo $dates
259}
260#==================================================================================
261function get_date_1 {
262# Assumes the following filename structure: *_${Yb}*-${Ye}*.* (Yb/e: 4 digits)
263# Returns ${FirstYear}-${LastYear}-${IntervalsLengths}
264# $1: varname    $2: datatype    $3: dirin    $4: <dir_in> patch    $5: <file_in>
265# [$6...]: <dset>:<var>=<val> to overwrite value of variables evaluated from parsing.
266  local v_in dtyp d_in d_str f_in Yb Ye St a f first
267  v_in=$1; shift; dtyp=$1; shift; d_in=$1; shift; dstr=$1; shift; f_in=$1; shift
268  parse_dir $f_in $dstr $*; Yb=1000000; Ye=0; first='t'
269  for f in $(var=$v_in; Yb=????; Ye=????; datatype=$dtyp; eval ls $d_in/$f_in); do
270    a=${f##*_}; ymn=${a::4}; if [ $ymn -le $Yb ]; then Yb=$ymn; fi
271    a=${a##*-}; ymx=${a::4}; if [ $ymx -ge $Ye ]; then Ye=$ymx; fi
272    if [ $first != 't' ]; then St=${St}-; fi; first='f'; St=${St}$((ymx-ymn+1))
273  done
274  echo $Yb-$Ye-$St
275}
276#==================================================================================
277function dim_len {
278  local n
279  if [ "$2" = "T" ]; then
280    n=$(ncdump -h $1 | grep 'UNLIMITED'); n=${n#*// (}; echo ${n% currently*}
281  else
282    n=$(ncdump -h $1 | grep $2' = '); n=${n#* = }; echo ${n% ;*}
283  fi
284}
285#==================================================================================
286function get_grid {
287  # $1: filename $2: varname
288  X=$(dim_nam $1 X); if [ $? -ne 0 ]; then X=""; else lX=$(dim_len $1 $X); fi
289  Y=$(dim_nam $1 Y); if [ $? -ne 0 ]; then Y=""; else lY=$(dim_len $1 $Y); fi
290  Z=$(dim_nam $1 Z); if [ $? -ne 0 ]; then Z=""; else lZ=$(dim_len $1 $Z); fi
291  T=$(dim_nam $1 T); if [ $? -ne 0 ]; then T=""; else lT=$(dim_len $1  T); fi
292  t=$(ncdump -h $1 | grep "double $2")
293  if [ $? -ne 0 ]; then t=$(ncdump -h $1 | grep "float $2"); fi
294  t=${t##*$2(}; t=${t%)*}; g=""; r=""
295  while :; do s=${t##*, }; t=${t%,*}
296    case $s in
297      $X) g=${g}X; r=${r}x$lX ;;
298      $Y) g=${g}Y; r=${r}x$lY ;;
299      $Z) g=${g}Z; r=${r}x$lZ ;;
300      $T) g=${g}T; r=${r}x$lT ;;
301    esac
302    if [ "$s" = "$t" ]; then break; fi
303  done
304  echo $g=${r:1:${#r}}
305}
306#==================================================================================
307function is_int {
308 # $1: string to be checked
309  set +vx
310  if [[ $1 =~ ^[0-9]+$ ]]; then return 0; else return 1; fi
311  set -vx
312}
313#==================================================================================
314function remove_version {
315 # $1: variable name
316  local out=$1
317  while `is_int ${out##*-}`; do out=${out%-*}; done; echo $out; return 1
318}
319
Note: See TracBrowser for help on using the repository browser.