Changeset 3870

Timestamp:

05/31/18 16:42:58 (5 years ago)

Author:

cugnet

Message:

Available datasets list on ciclad extended.
Genral tools contained in tools.bash are now fully independent of the main script clims_CMIP6.

Location:

TOOLS/CMIP6_FORCING/OZONE

Files:

• clims_CMIP6.bash (modified) (8 diffs)
• tools.bash (modified) (4 diffs)

TOOLS/CMIP6_FORCING/OZONE/clims_CMIP6.bash

@@ -3 +3 @@
 
 #=== Indices of chosen datasets among available collection (see DATANAMES after)
-DOPROCESS=('y' 'y')          #--- PROCESSED DATA TYPES (SSTandSeaIce, ozone)
-IXPROCESS=("0" "3")        #--- PROCESSED DATASETS INDICES
+DOPROCESS=('n' 'y')          #--- PROCESSED DATA TYPES (SSTandSeaIce, ozone)
+IXPROCESS=("0" "5 6 7")      #--- PROCESSED DATASETS INDICES
 recomp='n'                   #--- RECOMPUTE FILES THAT ARE ALREADY PRESENT (y/n)
 
@@ -31 +31 @@
 esac
 if [ "$DATAIN" = "" ]; then echo "Raw data folder for $machine has to be specified."; exit; fi
-DATAOU=/data/dcugnet$DATAOU
+# DATAOU=/data/dcugnet$DATAOU  # No write acces to $DATAOU => temporary storage space.
 
 #--- DATA TO TREAT (INTERANNUAL AND CLIMATOLOGIES): "SSTsAndSeaIce" "ozone":
@@ -42 +42 @@
            input4MIPs/CMIP6/CMIP/PCMDI/PCMDI-AMIP-1-1-4/ocean/mon/\${var}/gn/v20180427/$f" \
           "input4MIPs/CMIP6/CMIP/UReading/UReading-CCMI-1-0/atmos/mon/\${var}/gn/v20160711/$f   \
+           input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-stratO3-1-0/atmos/mon/\${var}/gn/v20170814/$f \
            input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-nat-1-0/atmos/mon/\${var}/gn/v20180503/$f     \
            input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-sol-1-0/atmos/mon/\${var}/gn/v20180503/$f     \
-           input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-stratO3-1-0/atmos/mon/\${var}/gn/v20170814/$f \
-           input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-volc-1-0/atmos/mon/\${var}/gn/v20180503/$f")
+           input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-volc-1-0/atmos/mon/\${var}/gn/v20180503/$f    \
+           input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-sol-1-1/atmos/mon/\${var}/gn/v20180525/$f     \
+           input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-nat-1-1/atmos/mon/\${var}/gn/v20180525/$f     \
+           input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-volc-1-1/atmos/mon/\${var}/gn/v20180525/$f")
 CLIMS_IN=("/ / / / /" \
           "input4MIPs/CMIP6/CMIP/UReading/UReading-CCMI-1-0/atmos/monC/\${var}/gn/v20160830/$f  /  /  /  /")
 VARNAM_IN=('tosbcs siconcbcs' 'vmro3')
 DIRSTRUCT='datasets/exercise/project/origin/dataset/domain/frequency/var/grid/version/'
-DATADATES=()                                #--- DETERMINED AUTOMATICALLY LATER
+DATADATES=()                                #--- DETERMINED LATER
 
 #--- RESULTS STORAGE
@@ -60 +63 @@
 VARNAMFOU=('sst sic'             'tro3')    #--- VARIABLES NAMES FOR FILES NAMES
 CLIMDATES=('1870-1899 1979-2008' '1850-1879 1979-2008')
-coords="X:longitude Y:latitude Z:plev"      #--- COORDINATES FOR ce0l
+coords="X:longitude Y:latitude Z:plev"      #--- COORDINATES NAMES KNOWN BY ce0l
+
+#=== EXCEPTIONS FOR STORAGE TREE ERRORS: variables not deduced from folder name parsing
+ex="PCMDI-AMIP-1-1-0:grid=gs1x1"  # <dset>:<var>=<val> if dataset=dset: export var=val
 
 #==================================================================================
-#=== FEW FUNCTIONS
-#   NOTE: These functions are not specific to this script.
-#   EXCEPTION: in get_date (error in ciclad storage tree for PCMDI-AMIP-1-1-0):
-#          if [ $dataset = "PCMDI-AMIP-1-1-0" ]; then grid='gs1x1'; fi
+#=== FEW FUNCTIONS (not specific to this script)
 #==================================================================================
 source tools.bash
-
 
 #=== DETERMINE TIME INTERVALS AND DISPLAY WHAT DATASET ARE ABOUT TO BE TREATED
 for i in $(eval echo {0..$((${#DATATYPES[@]}-1))}); do
   fin=${FILES_IN[$i]}; dt=${DATATYPES[$i]}; var=${VARNAM_IN[$i]%% *}
-  DATADATES[$i]=$(get_date $var $dt $DATAIN $fin); date=($DATADATES[$i]); fin=($fin)
-  if [ ${DOPROCESS[$i]} != 'y' ]; then continue; fi
+  DATADATES[$i]=$(get_date $var $dt $DATAIN $DIRSTRUCT $fin $ex)
+  date=(${DATADATES[$i]}); fin=($fin); if [ ${DOPROCESS[$i]} != 'y' ]; then continue; fi
   echo ">> Chosen $dt:"
   for ka in ${IXPROCESS[$i]}; do echo "  * ${fin[$ka]%/*} (${date[$ka]%-*})"; done
@@ -93 +95 @@
   for is in ${IXPROCESS[$i]}; do                           #=== LOOP ON VERSIONS
   #================================================================================
-    parse_dir ${fin[$is]} $DIRSTRUCT                       #--- Parse input directory
-    gridin=$grid; if [ $dataset = "PCMDI-AMIP-1-1-0" ]; then gridin='gs1x1'; fi
+    parse_dir ${fin[$is]} $DIRSTRUCT $ex                   #--- Parse input directory
     dso=$dataset                                           #--- For naming purpose
     case $dt in                                            #--- NAMING EXCEPTIONS
-      ozone) if [ "$dataset" =    "PCMDI-AMIP-1-0" ]; then dso='AMIP'; fi ;;
-      SSTAndSeaIce) dso=${dso#*hist-}; dso=hist-${dso%%-*}
-             if [ "$dataset" = "UReading-CCMI-1-0" ]; then dso="historical"; fi ;;
+      ozone) dso=${dso#*hist-}; dso=hist-${dso%%-*}
+                    if [ "$dataset" = "UReading-CCMI-1-0" ]; then dso="historical"; fi ;;
+      SSTAndSeaIce) if [ "$dataset" =  "PCMDI-AMIP-1-1-0" ]; then dso='AMIP'; fi ;;
     esac
     #==============================================================================
@@ -106 +107 @@
       da=${date[$is]}; Yb=${da%%-*}; Ye=${da#*-}; dY=${Ye##*-}; Ye=${Ye%-*}
       v_in=${vin[$iv]}; v_ou=${vou[$iv]}                   #--- Variables names
-      f_in=$(var=$v_in; Yb=\${Yb}; Ye=\${Ye}; datatype=$dt; grid=$gridin; eval echo $DATAIN/${fin[$is]})
+      f_in=$(var=$v_in; Yb=\${Yb}; Ye=\${Ye}; datatype=$dt; eval echo $DATAIN/${fin[$is]})
       axis=$(get_grid $(Yb=$Yb; Ye=????; eval echo $f_in) $v_in)
       res=${axis#*=}; axis=${axis%=*}
-      doZon='n'; if [ $axis = 'XYZT' ]; then doZon='y'; fi #--- Zonal mean ?  For 3D files
-      if [ $doZon = 'y' ]; then                            #--- 2 directories for 3D files
-        res3D=${res%x*}; res2D=${res3D#*x}_zonal; f_ou=${f_ou%/*}/\\\${res}/${f_ou##*/}
+      Zonal='n'; if [ $axis = 'XYZT' ]; then Zonal='y'; fi #--- Zonal mean ?  For 3D files
+      f_ou=$(var=${vfo[$iv]}; Y=\${Y}; dataset=$dso; eval echo $DATAOU/${fou})
+      if [ $Zonal = 'y' ]; then                            #--- 2 directories for 3D files
+        res3D=${res%x*}; res2D=${res3D#*x}_zonal; f_ou=${f_ou%/*}/\${res}/${f_ou##*/}
       fi
-      f_ou=$(var=${vfo[$iv]}; Y=\${Y}; dataset=$dso; eval echo $DATAOU/${fou})
       echo ">> BUILDING 12 MONTHS ${v_in} FILES FOR THE $Yb-$Ye PERIOD USING $dataset DATASET ($version)..."
       #============================================================================
@@ -123 +124 @@
           rename_dims $fO $coords                          #--- Rename dims/atts
         fi
-        if [ $doZon = 'y' ]; then                          #--- Zonal mean
+        if [ $Zonal = 'y' ]; then                          #--- Zonal mean
           fZ=$(Y=$Y; res=${res2D}; eval echo ${f_ou})      #--- Zonal File name
           if [[ ! -f $fZ || $recomp = 'y' ]]; then zonal_mean $fO $fZ; fi
@@ -136 +137 @@
 
         #=== Is there a single file climatology available ?
-        fC0=$(var=$v_in; Yb=$Yb; Ye=$Yb; datatype=$dt; eval echo $DATAIN/${cli0})
-        if [[ ${cli0} != "/" && -f $fC0 ]]; then c='y'; Yc=${Yb}_${Yb}_clim; fi
+        fc0=$(var=$v_in; Yb=$Yb; Ye=$Yb; datatype=$dt; eval echo $DATAIN/${cli0})
+        if [[ ${cli0} != "/" && -f $fc0 ]]; then c='y'; Yc=${Yb}_${Yb}_clim; fi
 
-        fO=$(Y=$Yc; res=${res3D}; eval echo ${f_ou})       #--- Original resolution
-        if [[ ! -f $fO || $recomp = 'y' ]]; then
+        fO=$(Y=\$Y; res=${res3D}; eval echo ${f_ou})       #--- Original resolution
+        fOc=$(Y=$Yc; res=${res3D}; eval echo ${f_ou})      #--- ", target climatology
+        if [[ ! -f $fOc || $recomp = 'y' ]]; then
           if [ "$c" = "y" ]; then
             echo ">> BUILDING $v_ou CLIMATOLOGY USING SPECIAL FILE ${fC0##*/} ($version)..."
-            cp $fC0 $fO ; rename_dims $fO $coords          #--- Rename dims/atts
+            cp $fc0 $fOc ; rename_dims $fOc $coords        #--- Rename dims/atts
           else
             echo ">> BUILDING $v_ou CLIMATOLOGY USING YEARLY FILES FOR $per ($version)..."
-            make_clim ${v_ou} $f_ou $per 0                 #--- Compute climatology
+            make_clim ${v_ou} $fO $per 0                   #--- Compute climatology
           fi
         fi
-        if [ $doZon = 'y' ]; then
-          fZ=$(Y=$Y; res=${res2D}; eval echo ${f_ou})      #--- Zonal File name
-          if [[ ! -f $fZ || $recomp = 'y' ]]; then zonal_mean $fO $fZ; fi
+        if [ $Zonal = 'y' ]; then
+          fZc=$(Y=$Yc; res=${res2D}; eval echo ${f_ou})    #--- Zonal File name
+          if [[ ! -f $fZc || $recomp = 'y' ]]; then zonal_mean $fOc $fZc; fi
         fi
       done

TOOLS/CMIP6_FORCING/OZONE/tools.bash

@@ -1 +1 @@
 function parse_dir {
-# Syntax: export variables from directory $1 with structure var1/var2/.../varN ($2)
-# Each variable ${vari} is exported.
-  local d=$1 s=$2
+# Syntax:   parse_dir <dir> <dir_struct> [<patch1 ... patchN>]
+#       $1:     directory to parse
+#       $2:     directory structure
+#      [$3...]: <dset>:<var>=<val> patches
+# Purpose: parse directory <dir> with structure <dir_struct> and export all variables.
+# If $dataset = $dset, the value of <var> obtain with parsing is replaced with <val>.
+  local d s
+  d=$1; shift; s=$1; shift
   while :; do
-    export ${s%%/*}=${d%%/*}
+    $(eval echo "export ${s%%/*}=${d%%/*}")
     s=${s#*/}; d=${d#*/}
     if [ "$s" = "${s#*/}" ]; then break; fi
   done
+  while [ $# -ge 1 ]; do if [ $dataset = ${1%:*} ]; then eval export ${1#*:}; fi; shift; done
+  unset d s
+}
+#==================================================================================
+function replace_1 {
+  # $1: string  $2: <patch>=<replacement string>
+  local out=$1
+  while [ "${out%${2%=*}*}" != "$out" ]; do out=${out%${2%=*}*}"${2#*=}"${out##*${2#=*}*}; done
+  echo $out; unset out; return 0
+}
+#==================================================================================
+function replace {
+  # same as "replace_1", but with several <patch>=<replacement string> arguments.
+  local out=$1; shift
+  while [ $# -ne 0 ]; do out=$(replace_1 $out "$1"); shift; done
+  echo $out; unset out; return 0
 }
 #==================================================================================
 function addc {
-  local s
-  s="$1"; while [ ${#s} -lt $2 ]; do s="_$s"; done ; echo "$s"; unset s
+  local s="$1"; while [ ${#s} -lt $2 ]; do s="_$s"; done ; echo "$s"; unset s
 }
 #==================================================================================
@@ -114 +134 @@
   done
   f_ou=$(var=$1; Y=${Yb}_${Ye}_clim; eval echo $2)
+  if [ ! -p ${f_ou%/*} ]; then mkdir -p ${f_ou%/*}; fi
   rm -f $f_ou ; ncrcat ${1}_??.nc $f_ou ; rm -f ${1}_??.nc
   f0=`var=$1; Y='YYYY'; eval echo $2`; f0=$(var=$1; Y='YYYY'; eval echo $f0); f0=${f0##*/}
   ncatted -a comment,global,a,c,"\nClimatology from $Yb to $Ye using file(s) $f0" $f_ou
-  echo ">> Done for variable $1."
 }
 #==================================================================================
@@ -218 +238 @@
 #    $1: input file  $2: output file
   local tmp=tmp.$$.nc
+  if [ ! -d ${2%/*} ]; then mkdir -p ${2%/*}; fi
   ncwa       -a longitude $1 $tmp ; mv $tmp $2    #--- Zonal mean
   ncks -O -x -v longitude $2 $tmp ; mv $tmp $2    #--- Remove longitude
@@ -225 +246 @@
 #==================================================================================
 function get_date {
-# $1: varname  $2: datatype  $3: dirin  $4:...: files_in
 # Assumes the following filename structure: *_${Yb}*-${Ye}*.* (Yb/e: 4 digits)
-  local v_in dtyp d_in f_in ixf Yb Ye St a f first dates
-  v_in=$1; shift; dtyp=$1; shift; d_in=$1; shift
-  for f_in in $@; do parse_dir ${f_in} $DIRSTRUCT; Yb=1000000; Ye=0; first='t'
-    #=== /!\ ERROR IN THE STORAGE TREE ON CICLAD FOR PCMDI-AMIP-1-1-0
-    if [ $dataset = "PCMDI-AMIP-1-1-0" ]; then grid='gs1x1'; fi
-    for f in $(var=$v_in; Yb=????; Ye=????; datatype=$dtyp; eval echo $d_in/$f_in); do
-      a=${f##*_}; if [ ${a::4} -le $Yb ]; then Yb=${a::4}; fi
-      a=${a##*-}; if [ ${a::4} -ge $Ye ]; then Ye=${a::4}; fi
-      if [ $first = 't' ]; then St=$((Ye-Yb+1)); first='f'; fi
-    done
-    if [ $f_in = $1 ]; then dates="$Yb-$Ye-$St"; else dates="$dates $Yb-$Ye-$St"; fi
+# $1: varname
+# $2: datatype
+# $3: dirin
+# $4: dirin structure patch
+# $5...$N: files_in
+# [$N+1...]: <dset>:<var>=<val> to overwrite value of variables evaluated from parsing.
+  local v_in dtyp d_in dstr f_in Yb Ye St a f first date dates files_in=""
+  v_in=$1; shift; dtyp=$1; shift; d_in=$1; shift; dstr=$1; shift; f0=$1
+  while [ "$1" = "${1##*=}" ]; do files_in="$files_in $1"; shift; done
+  for f_in in $files_in; do date=$(get_date_1 $v_in $dtyp $d_in $dstr $f_in $*)
+    if [ $f_in = $f0 ]; then dates=$date; else dates="$dates $date"; fi
   done
   echo $dates
+}
+#==================================================================================
+function get_date_1 {
+# Assumes the following filename structure: *_${Yb}*-${Ye}*.* (Yb/e: 4 digits)
+# $1: varname
+# $2: datatype
+# $3: base directory for files
+# $4: dirin structure patch
+# $5: file_in
+# [$6...]: <dset>:<var>=<val> to overwrite value of variables evaluated from parsing.
+  local v_in dtyp d_in d_str f_in Yb Ye St a f first
+  v_in=$1; shift; dtyp=$1; shift; d_in=$1; shift; dstr=$1; shift; f_in=$1; shift
+  parse_dir $f_in $dstr $*; Yb=1000000; Ye=0; first='t'
+  for f in $(var=$v_in; Yb=????; Ye=????; datatype=$dtyp; eval ls $d_in/$f_in); do
+    a=${f##*_}; if [ ${a::4} -le $Yb ]; then Yb=${a::4}; fi
+    a=${a##*-}; if [ ${a::4} -ge $Ye ]; then Ye=${a::4}; fi
+    if [ $first = 't' ]; then St=$((Ye-Yb+1)); first='f'; fi
+  done
+  echo $Yb-$Ye-$St
 }
 #==================================================================================