Changeset 8859

Timestamp:

2017-11-30T16:01:29+01:00 (7 years ago)

Author:

nicolasmartin

Message:

Continuation of global refactoring of Trusting tool

• Introduce new 'dev' mode beetween 'debug' and 'prod' modes to skip XIOS compilation from scratch and working in a solely testing directory (thanks Andrew for the suggestion)
• Switch cfg files from 'svn:mime-type=text/x-shellscript' to 'svn:executable' trying to get syntax highlighting under Trac
• Bugfixes: correct XIOS branch 'XIOS_DEV_CMIP6' for ORCA1 and remove remaining 'TRUST_DIR_BENCHMARK' (replace by TRUST_TEST_BENCHMARK)
• Improve the workflow displayed in the terminal

Location:

branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST

Files:

• cfg/X64_ADA.cfg (modified) (2 props)
• cfg/X64_CURIE.cfg (modified) (2 props)
• cfg/X64_CURIE_trust.cfg (modified) (2 props)
• cfg/arch_template.cfg (modified) (2 props)
• cfg/martin.cfg (modified) (2 diffs, 2 props)
• cfg/romr005.cfg (modified) (2 props)
• cfg/user_template.cfg (modified) (2 props)
• inc/trusting.env (modified) (2 diffs)
• inc/trusting_func.sh (modified) (6 diffs, 2 props)
• trusting.sh (modified) (8 diffs, 2 props)

branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/cfg/martin.cfg

@@ -24 +24 @@
      TRUST_IO_XIOS=$CCCWORKDIR/XIOS
 
-if   [ ${TRUST_SVN_BRANCH} == 'nemo_v3_6_STABLE'  ]; then
-    TRUST_IO_XIOS+='/xios-1.0'
-elif [ ${TRUST_CFG_REF} == 'ORCA1_LIM3_PISCES'    ]; then
-    TRUST_IO_XIOS+='/XIOS_DEV_CMIP6'
+if [ ${TRUST_SVN_BRANCH} == 'nemo_v3_6_STABLE'  ]; then
+
+    if [ ${TRUST_CFG_REF} == 'ORCA1_LIM3_PISCES' ]; then
+   TRUST_IO_XIOS+='/XIOS_DEV_CMIP6'
+    else
+   TRUST_IO_XIOS+='/xios-1.0'
+    fi
+
 else
     TRUST_IO_XIOS+='/trunk'
@@ -36 +40 @@
 ##--------------
 
-TRUST_TEST_MAILING=$( cat ./cfg/mailing_list.txt )
+TRUST_TEST_MAILING=$( < ./cfg/mailing_list.txt )

branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/inc/trusting.env

@@ -12 +12 @@
 ##-------------------------------------------------------------------
 
-if [ ${TRUST_FLAG_DEBUG} == 'false' ]; then
-    ## std mode (merge stdout & stderr to /dev/null, XIOS compilation from scratch)
+if   [ ${TRUST_FLAG_DEBUG} == 'true' ]; then
+    ## debug mode (skip XIOS compilation    , output stdout & stderr)
+    TRUST_IO_XIOS_MODE=''
+    TRUST_MAIN_STDOUT=''
+elif [ ${TRUST_FLAG_DEV}   == 'true' ]; then
+    ## dev mode   (skip XIOS compilation    , output only stderr    )
+    TRUST_IO_XIOS_MODE=''
+    TRUST_MAIN_STDOUT='1> /dev/null'
+else
+    ## prod mode  (compile XIOS from scratch, no stdout nor stderr  )
     TRUST_IO_XIOS_MODE='--full'
     TRUST_MAIN_STDOUT='>& /dev/null'
-else
-    ## debug mode (verbose output and skip XIOS compilation)
-    TRUST_IO_XIOS_MODE=''
-    TRUST_MAIN_STDOUT='1> /dev/null'
 fi
 
@@ -208 +212 @@
 fi
 
-export TRUST_TEST_LOG=${TRUST_DIR_BENCHMARK}/trusting_${TRUST_TEST_NAME}.txt
+export TRUST_TEST_LOG=${TRUST_TEST_BENCHMARK}/trusting_${TRUST_TEST_NAME}.txt
 
 export TRUST_TEST_MAILING

branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/inc/trusting_func.sh

@@ -18 +18 @@
 ##--------------------------------------------------------------------------------
 
-print_step() {
+step() {
     local char_nb=$( echo "$1" | wc -c )
     local outline=$( printf "%${char_nb}s" )
 
-    printf "\n\nStep.....\n%s\n%s\n\n" "$1" ${outline// /-}
+    printf "\n%s\n%s\n\n" "$1" ${outline// /-}
 }
 
@@ -152 +152 @@
     cd ${TRUST_SVN_NEMOGCM}/CONFIG
 
-    ## Recompiling from scratch if not in debug mode
-    if [[ -d ${TRUST_CFG_NEW} && ${TRUST_FLAG_DEBUG} == 'false' ]]; then
-   ./makenemo -n ${TRUST_CFG_NEW} clean_config \
-       > /dev/null <<EOF
+    ## Recompiling from scratch if not in debug or dev mode
+    if [[ ${TRUST_FLAG_DEBUG} == 'false' && ${TRUST_FLAG_DEV} == 'false' ]]; then
+
+     if [[ -d ${TRUST_CFG_NEW} ]]; then
+         ./makenemo -n ${TRUST_CFG_NEW} clean_config \
+         > /dev/null <<EOF
 y
 EOF
+     fi
+
     fi
 
@@ -420 +424 @@
        if [ -e ${TRUST_TEST_BENCHMARK}/$filebase.nc ]; then
 
-      #bcmk='true'
       cdo diffn $filebase.nc ${TRUST_TEST_BENCHMARK}/$filebase.nc \
           > cdo_diff.out 2> /dev/null
@@ -574 +577 @@
 
        ## Computing
-       'H') TRUST_FLAG_ERROR='Crashed at time step'              ;;
-       'I') TRUST_FLAG_ERROR='Exceeded time limit'               ;;
+       'H') TRUST_FLAG_ERROR='Crashed at time step '
+      comments 'E R R O R'
+      [ -e time.step ] && time_step=$( cat time.step )
+      TRUST_FLAG_ERROR+=${time_step:=0}                     ;;
+       'I') TRUST_FLAG_ERROR='Exceeded time limit of '
+      TRUST_FLAG_ERROR+=$(( ${TRUST_JOB_TIMEOUT}/3600 ))'h' ;;
 
        ## Results
@@ -592 +599 @@
 
     ## Eventual comments from ocean.output
-    if [ ${TRUST_FLAG_ERROR} == 'Crashed at time step' ]; then
-   comments 'E R R O R'
-   [ -e time.step ] && time_step=$( cat time.step )
-   TRUST_FLAG_ERROR+=' '${time_step:=0}
-    else
-   comments 'W A R N I N G'
-
-   if [ ${TRUST_FLAG_ERROR} == 'Exceeded time limit' ]; then
-       TRUST_FLAG_ERROR+=' '$(( ${TRUST_JOB_TIMEOUT}/3600 ))'h'
-   fi
-
-    fi
+    [[ ! ${TRUST_FLAG_ERROR} =~ 'Crashed at time step' ]] && comments 'W A R N I N G'
 
     ## Last messenger files
@@ -609 +605 @@
     sed -i "2 s/.*/$TRUST_FLAG_ERROR/"  ${file_stat}
 
-    ## Save tested configuration if trusting failed in production mode (-p|--prod)
+    ## Save tested configuration if trusting failed in production mode ('-p')
     if [[ ${TRUST_FLAG_RESULT} == 'FAILED' && ${TRUST_FLAG_PROD} == 'true' ]]; then
    echo 'Creating archive '${TRUST_TEST_BACKUP}' under '${TRUST_TEST_BENCHMARK}

branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/trusting.sh

@@ -9 +9 @@
 
 TRUST_MAIN_DIR=$PWD
+TRUST_FLAG_HELP='false'
+TRUST_FLAG_DEBUG='false'; TRUST_FLAG_DEV='false' ; TRUST_FLAG_PROD='false'
 
-TRUST_FLAG_DEBUG='false'  ; TRUST_FLAG_HELP='false' ; TRUST_FLAG_PROD='false'
-
+# Revision number of the tool
 rev=$( svn info | awk '/Last Changed Rev/ {print $NF}' )
 
 
 ##--------------------------------------------------------------------------------
-## Get arguments from command line
+## Get short or long arguments from command line
 ##--------------------------------------------------------------------------------
 
@@ -45 +46 @@
     cat ./inc/trusting_help.txt
 
+    ## Color this section to part with help
+    printf "\033[0;33m"
+
     if [ ${TRUST_FLAG_HELP} == 'false' ]; then
-   printf "\n\n\033[0;33m"
-   printf "At least one configuration (arch or user) file is missing or misspelled:"
-   printf "\t'%s'.cfg\t'%s'.cfg" ${TRUST_MAIN_USER} ${TRUST_MAIN_HPCC}
-   printf "\033[0m"
+   printf "                                                           \
+       \nAt least 1 cfg file (arch or user) is missing or misspelled: \
+       '%s.cfg' or '%s.cfg'\n"                                        \
+       ${TRUST_MAIN_USER} ${TRUST_MAIN_HPCC}
     fi
 
-    printf "\n\nContent of 'config' folder:"
-    find config -name *.cfg | cut -d/ -f2 \
-   | xargs -n 4 printf "%-25s\t%-25s\t%-25s\n"
+    echo -e "\nContent of './cfg' folder:"
+    find ./cfg -name *.cfg | cut -d/ -f3 \
+   | xargs -n 3 printf "%-20s\t%-20s\t%-20s\n"
+    printf "\033[0m\n"
+
     exit 1
 else
@@ -95 +101 @@
 ##------------------
 
-print_step 'Timestamped testing directory'
+step 'Testing directory'
 init
 
@@ -102 +108 @@
 ##-------------------
 
-print_step 'SVN action on NEMO directories'
-printf "${TRUST_SVN_ACTION} on ${TRUST_SVN_NEMOGCM}:\n\n"  
+step 'Local working copy'
+
+echo "${TRUST_SVN_ACTION} on ${TRUST_SVN_NEMOGCM}:"
 get_nemo_rev
 
@@ -110 +117 @@
 ##------------
 
-print_step 'Get testing environment'
+step 'Environment'
 get_soft_rel
 
@@ -117 +124 @@
 ##---------------
 
-print_step 'Compile XIOS'
+step 'Compilation(s)'
+
+printf "XIOS? "
 compile_xios
 
-print_step "Compile ${TRUST_CFG_NEW} cfg from ${TRUST_CFG_REF}"
+printf "\nNEMO ${TRUST_CFG_NEW} cfg. from ${TRUST_CFG_REF}? "
 compile_nemo
 
@@ -127 +136 @@
 ##-------
 
-print_step 'Set job (copying or extracting inputs)'
+step 'Inputs'
+
 get_inputs
 
-print_step 'Compare inputs'
+printf "\nCompare with benchmark? "
 diff_inputs
 
@@ -137 +147 @@
 ##----
 
-print_step 'Submit job'
+step 'Job'
+
+printf "Submit? "
 job_submit
 
-print_step 'Pending job'
+echo 'Pending...'
 job_pending
 
-print_step 'Job finished'
+echo 'Finished!'
 
-print_step 'Test job state'
+printf "State? "
 job_state
 
-print_step 'Get job performances'
+printf "Perfs?"
 job_perfs
 
 
-## Outputs
-##--------
+## Output(s)
+##----------
 
-print_step 'Compare outputs'
+step 'Ouput(s)'
+
+echo 'Compare with benchmark:'
+
+printf "Plain text files? "
 diff_results
 
-print_step 'Compare restarts'
+printf "Restart files? "
 diff_restarts