Changeset 5455


Ignore:
Timestamp:
2015-06-19T22:57:06+02:00 (5 years ago)
Author:
nicolasmartin
Message:

dev_r5092_CNRS_SETTE Ping pong updates between Curie & Ada to finalize

Location:
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh

    r5453 r5455  
    11#!/bin/bash 
    2 # 
    3 # Author: Patrick Brockmann 
    4 # Contact: Patrick.Brockmann@cea.fr 
    5 # History: 
    6 # Modification: Nicolas-thierry.Martin@locean-ipsl.upmc.fr 
    7 # 
    82 
    93 
     
    115#--------------------------------------------------- 
    126NEMO_TRUS=$( pwd $( dirname $0 ) ) 
    13 DEBUG=0; PUBLISH=0; STATUS='OK' 
     7DEBUG=0; PUBLISH=0 
     8STATUS='FAILED'; STEP=''# Only at end status will change to OK 
    149 
    1510# Get options for replacing some initials settings 
     
    5146datestr1=$( LC_ALL=C date -d"${DATE_STR}" +%Y-%m-%dT%H:%M ) 
    5247printf "Date\n$datestr1\n" > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt 
    53 # Only at end status will change to OK 
    54 STATUS='FAILED'; STEP='' 
    5548 
    5649# SVN action on XIOS & NEMO essentials directories 
    5750#--------------------------------------------------- 
    58 print_step "Get NEMO SVN state with: ${SVN_OPT}" 
     51print_step 'Get NEMO code state / repository' 
    5952get_nemo_rev 
    6053 
     
    7063print_step 'Compiling XIOS' 
    7164cd ${DIR_XIOS} 
    72 #./make_xios --arch $ARCH -job $NPROC >& /dev/null 
     65./make_xios --full --arch $ARCH -job $NPROC #>& /dev/null 
     66[ ! -e ${DIR_XIOS}/lib/libxios.a ] && get_out $STATUS $STEP 
    7367 
    7468# NEMO config compiled from scratch 
     
    7670print_step "Compiling ${CONF_REF} configuration" 
    7771cd ${NEMO_CONF} 
    78 #[ -d ${TEST_CONF} ] && ./makenemo -n ${TEST_CONF} clean 
    79 ./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m $ARCH -j $NPROC >& /dev/null 
     72[ -d ${TEST_CONF} ] && ./makenemo -n ${TEST_CONF} clean 
     73./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m $ARCH -j $NPROC #>& /dev/null 
    8074[ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out $STATUS $STEP 
    8175 
    8276# Get namelists, xml & forced files for running 
    8377#--------------------------------------------------- 
    84 print_step 'Preparing job' 
     78print_step "Preparing job by copying ${CONF_REF} input files" 
    8579cd ${TEST_DIR} 
    8680find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \; 
     
    9488${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null 
    9589[ $? -ne 0 ] && get_out $STATUS $STEP 
     90 
    9691[ $( find . -name '*.gz' -print -quit ) ] && gunzip *.gz 
    9792 
    9893# Check input files in all cases 
    9994#--------------------------------------------------- 
    100 print_step 'Test input files difference' 
     95print_step 'Difference between input files' 
    10196diff_inputs 
    10297 
    10398# Job submission & computing 
    10499#--------------------------------------------------- 
    105 print_step 'Running' 
     100print_step 'Submitting job' 
    106101JOB_ID=$( ${JOB_SUBM} | awk '{print $NF}' ) 
    107 export JOB_ID 
    108102[ $? -ne 0 ] && get_out $STATUS $STEP 
    109103 
     104print_step 'Pending/Running job' 
    110105time_elapsed=0; time_increment=30 
    111106sleep ${time_increment} 
     
    118113sleep ${time_increment} 
    119114 
     115if (( ${time_elapsed} = ${TIME_LIMI} )); then 
     116    print_step 'Killing job' 
     117    ${JOB_DELE} ${JOB_ID} &> /dev/null 
     118    get_out $STATUS 'Job timeout' 
     119fi 
     120print_step 'Job finished' 
     121 
    120122# Eventual crash during computing 
    121123#--------------------------------------------------- 
    122 print_step 'Test if ASCII output files exist' 
     124print_step 'Test if job crashed' 
    123125if   [[ ! -e ocean.output || $( grep 'E R R O R' ocean.output ) ]]; then 
    124     ${JOB_DELE} ${JOB_ID} &> /dev/null 
    125     [ ! -e ocean.output                  ] && STEP='nemo.exe crashed at initialization' 
    126     [ $( grep 'E R R O R' ocean.output ) ] && STEP='nemo.exe crashed' 
    127126    comments 'E R R O R' 
    128     ${JOB_DELE} ${JOB_ID} >/dev/null 2>&1 
    129127    get_out $STATUS $STEP 
    130128else 
     
    136134    for file in ocean.output solver.stat tracer.stat; do 
    137135   diff $file ${REFE_DIR}/$file 
    138    [ $? -ne 0 ] && STATUS='FAILED' && STEP='ASCII output file differ' 
     136   [ $? -ne 0 ] && get_out 'FAILED' 'ASCII output file differ' 
    139137    done 
    140138 
     
    142140    #--------------------------------------------------- 
    143141    print_step 'Test last restart NetCDF files diff' 
    144     diff_restart #2> /dev/null 
    145     [ $? -ne 0 ] && STATUS='FAILED' 
     142    diff_restart 
     143    [ $? -ne 0 ] && get_out 'FAILED' "$CDO diff failed" 
    146144 
    147145    # Get computation duration 
  • branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh

    r5453 r5455  
    33 
    44comments() { 
    5     [ -e ocean.output ] && line=$( grep $1 ocean.output ) 
     5    state=$1 
    66 
    7     # Check environment architecture modification 
    8     #--------------------------------------------------- 
    9     [ $( diff -q env.log ${REFE_DIR}/env.log ) ] && line=$( diff env.log ${REFE_DIR}/env.log ) 
     7    if [ $state == 'E R R O R' ]; then 
     8   if [ -e ocean.output ]; then 
     9       last_time_step=$( cat ${TEST_DIR}/time.step | tr -d [:space:] ) 
     10       STEP='nemo.exe crashed at '${last_time_step} && export STEP 
     11       line=$( grep -m1 "$state" ocean.output | tr -d '\n' ) 
     12   else 
     13       STEP='nemo.exe crashed at initialization'    && export STEP 
     14       return 
     15   fi 
     16    fi 
     17 
     18    if [ $( diff -q env.log ${REFE_DIR}/env.log ) ]; then 
     19   line=$( diff env.log ${REFE_DIR}/env.log  | tr -d '\n' ) 
     20    fi 
    1021 
    1122    printf "Comments\n$line\n" | tee mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt 
     
    4758       cmp ${TEST_DIR}/${base_name}_$comp.nc ${REFE_DIR}/${base_name}_$comp.nc 
    4859       if [ $? -ne 0 ]; then 
    49       export STEP='Restart files different' 
     60      STATUS='FAILED'; STEP='Restart files different' 
    5061      printf "$CDO diffn\n" 
    5162      $CDO diffn ${TEST_DIR}/${base_name}_$comp.nc ${REFE_DIR}/${base_name}_$comp.nc 2> /dev/null 
     
    5869 
    5970    else 
     71   STATUS='FAILED'; STEP='No restart files found at last time step' 
    6072   printf "no restart files found for comparison\n" 
    6173    fi 
     74 
     75    [ $STATUS == 'FAILED'] && get_out $STATUS $STEP 
    6276} 
    6377 
     
    93107   # For time being, just get revision number from XIOS 
    94108   if [ $dir == ${DIR_XIOS} ]; then 
    95        echo 'XIOS '$( svn info $dir | awk '(NR == 9) {print $NF}' ) >> env.log 
     109       echo 'XIOS '$( svn info $dir | awk '(NR == 5) {print $NF}' ) >> env.log 
    96110       continue 
    97111   fi 
     
    119133 
    120134  # Send mail only when FAILED 
    121   if [[ ! -z $MAIL && "$STATUS"=='FAILED' ]]; then 
     135  if [[ ! -z $MAIL && $STATUS == 'FAILED' ]]; then 
    122136      cat << END_MAIL > trusting.mail 
    123137XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 
     
    144158 
    145159    # Save tested configuration if trusting failed 
    146     if [ "$STATUS" == 'FAILED' ]; then 
     160    if [ $STATUS == 'FAILED' ]; then 
     161   cd ${TEST_DIR} 
    147162   touch mesg_08_inputfiles_${CFG_USER}_${CFG_ARCH}.txt mesg_09_realcputime_${CFG_USER}_${CFG_ARCH}.txt \ 
    148163         mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt 
Note: See TracChangeset for help on using the changeset viewer.