Changeset 4109 for TOOLS

Timestamp:

11/06/18 17:14:39 (5 years ago)

Author:

cugnet

Message:

Available datasets list on ciclad extended.
Scripts slightly modified to accomodate more ways of cutting a time serie into several time slices.

Location:

TOOLS/CMIP6_FORCING/OZONE

Files:

• clims_CMIP6.bash (modified) (10 diffs)
• tools.bash (modified) (5 diffs)

TOOLS/CMIP6_FORCING/OZONE/clims_CMIP6.bash

@@ -4 +4 @@
 #=== Indices of chosen datasets among available collection (see DATANAMES after)
 DOPROCESS=('n' 'y')          #--- PROCESSED DATA TYPES (SSTandSeaIce, ozone)
-IXPROCESS=("0" "5 6 7")      #--- PROCESSED DATASETS INDICES
+IXPROCESS=("0" "1 2 3 4")    #--- PROCESSED DATASETS INDICES
 recomp='n'                   #--- RECOMPUTE FILES THAT ARE ALREADY PRESENT (y/n)
 
@@ -31 +31 @@
 esac
 if [ "$DATAIN" = "" ]; then echo "Raw data folder for $machine has to be specified."; exit; fi
-# DATAOU=/data/dcugnet$DATAOU  # No write acces to $DATAOU => temporary storage space.
+ DATAOU=/data/dcugnet$DATAOU  # No write acces to $DATAOU => temporary storage space.
 
 #--- DATA TO TREAT (INTERANNUAL AND CLIMATOLOGIES): "SSTsAndSeaIce" "ozone":
@@ -42 +42 @@
            input4MIPs/CMIP6/CMIP/PCMDI/PCMDI-AMIP-1-1-4/ocean/mon/\${var}/gn/v20180427/$f" \
           "input4MIPs/CMIP6/CMIP/UReading/UReading-CCMI-1-0/atmos/mon/\${var}/gn/v20160711/$f   \
+           input4MIPs/CMIP6/ScenarioMIP/UReading/UReading-CCMI-ssp126-1-0/atmos/mon/\${var}/gn/v20181101/$f \
+           input4MIPs/CMIP6/ScenarioMIP/UReading/UReading-CCMI-ssp245-1-0/atmos/mon/\${var}/gn/v20181101/$f \
+           input4MIPs/CMIP6/ScenarioMIP/UReading/UReading-CCMI-ssp370-1-0/atmos/mon/\${var}/gn/v20181101/$f \
+           input4MIPs/CMIP6/ScenarioMIP/UReading/UReading-CCMI-ssp585-1-0/atmos/mon/\${var}/gn/v20181101/$f \
            input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-stratO3-1-0/atmos/mon/\${var}/gn/v20170814/$f \
            input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-nat-1-0/atmos/mon/\${var}/gn/v20180503/$f     \
@@ -48 +52 @@
            input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-sol-1-1/atmos/mon/\${var}/gn/v20180525/$f     \
            input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-nat-1-1/atmos/mon/\${var}/gn/v20180525/$f     \
-           input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-volc-1-1/atmos/mon/\${var}/gn/v20180525/$f")
+           input4MIPs/CMIP6/DAMIP/CCCma/CCMI-hist-volc-1-1/atmos/mon/\${var}/gn/v20180525/$f    \
+           input4MIPs/CMIP6/DAMIP/CCCma/CCMI-ssp245-nat-1-0/atmos/mon/\${var}/gn/v20180820/$f")
 CLIMS_IN=("/ / / / /" \
-          "input4MIPs/CMIP6/CMIP/UReading/UReading-CCMI-1-0/atmos/monC/\${var}/gn/v20160830/$f  /  /  /  /")
+          "input4MIPs/CMIP6/CMIP/UReading/UReading-CCMI-1-0/atmos/monC/\${var}/gn/v20160830/$f  /  /  /  / /")
 VARNAM_IN=('tosbcs siconcbcs' 'vmro3')
 DIRSTRUCT='datasets/exercise/project/origin/dataset/domain/frequency/var/grid/version/'
@@ -79 +84 @@
   date=(${DATADATES[$i]}); fin=($fin); if [ ${DOPROCESS[$i]} != 'y' ]; then continue; fi
   echo ">> Chosen $dt:"
-  for ka in ${IXPROCESS[$i]}; do echo "  * ${fin[$ka]%/*} (${date[$ka]%-*})"; done
+  for ka in ${IXPROCESS[$i]}; do d=${date[$ka]}; dy=${d%%-*}; d=${d#*-}
+    echo "  * ${fin[$ka]%/*} (${dy}-${d%%-*})"
+  done
 done
 
@@ -87 +94 @@
   if [ ${DOPROCESS[$i]} != 'y' ]; then continue; fi
      dt=${DATATYPES[$i]}                                   #=== SSTAndSeaIce, ozone...
-    fin=(${FILES_IN[$i]}); vin=(${VARNAM_IN[$i]})          #=== INPUT FILE/VAR NAMES (each version)
+    fin=(${FILES_IN[$i]}); vin=(${VARNAM_IN[$i]})          #===  INPUT FILE/VAR NAMES (each version)
      fou=${FILES_OU[$i]} ; vou=(${VARNAM_OU[$i]})          #=== OUTPUT FILE/VAR NAMES (single pair)
   date=(${DATADATES[$i]})                                  #=== ${Yb}-${Ye}-${dY}
@@ -96 +103 @@
   #================================================================================
     parse_dir ${fin[$is]} $DIRSTRUCT $ex                   #--- Parse input directory
-    dso=$dataset                                           #--- For naming purpose
-    case $dt in                                            #--- NAMING EXCEPTIONS
-      ozone) dso=${dso#*hist-}; dso=hist-${dso%%-*}
-                    if [ "$dataset" = "UReading-CCMI-1-0" ]; then dso="historical"; fi ;;
-      SSTAndSeaIce) if [ "$dataset" =  "PCMDI-AMIP-1-1-0" ]; then dso='AMIP'; fi ;;
-    esac
+    dso=$(remove_version $dataset); dso=${dso#*-}          #--- For naming purpose
+    if [ $dso = "CCMI" ]; then dso="historical"; fi        #--- NAMING EXCEPTION: UReading-CCMI-1-0
     #==============================================================================
     for iv in $(eval echo {0..$((${#vin[@]}-1))}); do      #--- Loop on variables
     #==============================================================================
-      da=${date[$is]}; Yb=${da%%-*}; Ye=${da#*-}; dY=${Ye##*-}; Ye=${Ye%-*}
+      da=${date[$is]}; Yi=${da%%-*}; Yf=${da#*-}; dYt=${Yf#*-}; Yf=${Yf%%-*}
       v_in=${vin[$iv]}; v_ou=${vou[$iv]}                   #--- Variables names
       f_in=$(var=$v_in; Yb=\${Yb}; Ye=\${Ye}; datatype=$dt; eval echo $DATAIN/${fin[$is]})
-      axis=$(get_grid $(Yb=$Yb; Ye=????; eval echo $f_in) $v_in)
+      axis=$(get_grid $(Yb=$Yi; Ye=????; eval echo $f_in) $v_in)
       res=${axis#*=}; axis=${axis%=*}
       Zonal='n'; if [ $axis = 'XYZT' ]; then Zonal='y'; fi #--- Zonal mean ?  For 3D files
@@ -115 +118 @@
         res3D=${res%x*}; res2D=${res3D#*x}_zonal; f_ou=${f_ou%/*}/\${res}/${f_ou##*/}
       fi
-      echo ">> BUILDING 12 MONTHS ${v_in} FILES FOR THE $Yb-$Ye PERIOD USING $dataset DATASET ($version)..."
+      echo ">> BUILDING 12 MONTHS ${v_in} FILES FOR THE $Yi-$Yf PERIOD USING $dataset DATASET ($version)..."
+      Ye=$((Yi-1))
       #============================================================================
-      for Y in $(eval echo {$Yb..$Ye}); do                 #--- Loop on years
+      for Y in $(eval echo {$Yi..$Yf}); do                 #--- Loop on years
       #============================================================================
         fO=$(Y=$Y; res=${res3D}; eval echo ${f_ou})        #--- Original resolution
+        if [ $Y -eq $((Ye+1)) ]; then Yb=$Y; dY=${dYt%%-*}; Ye=$((Yb+dY-1)); dYt=${dYt#*-}; fi
         if [[ ! -f $fO || $recomp = 'y' ]]; then
           extract $v_in,$v_ou $f_in,$fO $Yb,$Ye,$dY $Y 0   #--- Field extraction
@@ -128 +133 @@
           if [[ ! -f $fZ || $recomp = 'y' ]]; then zonal_mean $fO $fZ; fi
         fi
-        progress_bar $((Y-Yb+1)) $((Ye-Yb+1)) 50           #--- Check progress
+        progress_bar $((Y-Yi+1)) $((Yf-Yi+1)) 50           #--- Check progress
       #============================================================================
       done
@@ -135 +140 @@
       #============================================================================
         Yb=${per%%-*}; Ye=${per##*-}; Yc=${Yb}_${Ye}_clim; c='n'; cli0=${cli[$is]}
+
+        #=== Skip periods outside available years interval
+        if [[ $Yb -lt ${da%%-*} || ${Yb} -gt ${da#*-} || $Yb -lt ${da%%-*} || ${Yb} -gt ${da#*-} ]]; then
+          continue
+        fi
 
         #=== Is there a single file climatology available ?

TOOLS/CMIP6_FORCING/OZONE/tools.bash

@@ -76 +76 @@
     Yb=$Ymin; Ye=$Ymax
   else
-    Yb=$((dY*(Y/dY))); Ye=$((Yb+dY-1)); Ye=$(($Ye>$Ymax?$Ymax:$Ye))
+    Yb=$((Ymin+dY*((Y-Ymin)/dY))); Ye=$((Yb+dY-1))
+    Ye=$(($Ye>$Ymax?$Ymax:$Ye))
   fi
   nbm=$((12*(Ye-Yb+1)-1))                           #--- MONTHS NUMBER OF CURRENT FILE
   it=$((12*(Y-Yb)))                                 #--- JANUARY INDEX IN CURRENT FILE
   fin=`var=$v_in; Yb=$Yb; Ye=$Ye; eval echo $f_in`  #--- CURRENT FILE NAME
-  fou=`var=$v_ou; Y=$Y; eval echo $f_ou`            #--- OUTPUT FILE NAME
+  fou=`var=$v_ou; Y=$Y; eval echo $f_ou`            #--- OUTPUT  FILE NAME
   ib=$(($it<=nmrg?0:$((it-nmrg))))                  #--- INDEX OF FIRST USED MONTH
   ie=$(($((it+11+mrg))>=$nbm?$nbm:$((it+11+nmrg)))) #--- INDEX OF LAST USED MONTHF
@@ -246 +247 @@
 #==================================================================================
 function get_date {
-# Assumes the following filename structure: *_${Yb}*-${Ye}*.* (Yb/e: 4 digits)
-# $1: varname
-# $2: datatype
-# $3: dirin
-# $4: dirin structure patch
-# $5...$N: files_in
+# Call get_date1 one time each input file.
+# $1: varname    $2: datatype    $3: dirin    $4: <dir_in> patch    $5...$N: <files_in>
 # [$N+1...]: <dset>:<var>=<val> to overwrite value of variables evaluated from parsing.
   local v_in dtyp d_in dstr f_in Yb Ye St a f first date dates files_in=""
@@ -264 +261 @@
 function get_date_1 {
 # Assumes the following filename structure: *_${Yb}*-${Ye}*.* (Yb/e: 4 digits)
-# $1: varname
-# $2: datatype
-# $3: base directory for files
-# $4: dirin structure patch
-# $5: file_in
+# Returns ${FirstYear}-${LastYear}-${IntervalsLengths}
+# $1: varname    $2: datatype    $3: dirin    $4: <dir_in> patch    $5: <file_in>
 # [$6...]: <dset>:<var>=<val> to overwrite value of variables evaluated from parsing.
   local v_in dtyp d_in d_str f_in Yb Ye St a f first
@@ -274 +268 @@
   parse_dir $f_in $dstr $*; Yb=1000000; Ye=0; first='t'
   for f in $(var=$v_in; Yb=????; Ye=????; datatype=$dtyp; eval ls $d_in/$f_in); do
-    a=${f##*_}; if [ ${a::4} -le $Yb ]; then Yb=${a::4}; fi
-    a=${a##*-}; if [ ${a::4} -ge $Ye ]; then Ye=${a::4}; fi
-    if [ $first = 't' ]; then St=$((Ye-Yb+1)); first='f'; fi
+    a=${f##*_}; ymn=${a::4}; if [ $ymn -le $Yb ]; then Yb=$ymn; fi
+    a=${a##*-}; ymx=${a::4}; if [ $ymx -ge $Ye ]; then Ye=$ymx; fi
+    if [ $first != 't' ]; then St=${St}-; fi; first='f'; St=${St}$((ymx-ymn+1))
   done
   echo $Yb-$Ye-$St
@@ -311 +305 @@
 }
 #==================================================================================
+function is_int {
+ # $1: string to be checked
+  set +vx
+  if [[ $1 =~ ^[0-9]+$ ]]; then return 0; else return 1; fi
+  set -vx
+}
+#==================================================================================
+function remove_version {
+ # $1: variable name
+  local out=$1
+  while `is_int ${out##*-}`; do out=${out%-*}; done; echo $out; return 1
+}