- Timestamp:
- 2015-06-04T20:09:56+02:00 (9 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh
r5292 r5355 9 9 10 10 # Set defaults 11 #--------------------------------------------------- 12 NEMO_TRUS=$( pwd $( dirname $0 ) ) 11 13 DEBUG=0; PUBLISH=0; status='OK' 12 14 13 # Get options to replace some initials settings 15 # Get options for replacing some initials settings 16 #--------------------------------------------------- 14 17 while [ $# -ne 0 ]; do 15 18 case $1 in 16 19 -b|--branch ) NEMO_BRAN=$2 ; shift 2;; -d|--debug ) set -vx; DEBUG=1 ; shift ;; 17 20 -e|--email ) MAIL=$2 ; shift 2;; -j|--job ) NPROC=$2 ; shift 2;; 18 -h|--help ) cat trusting_help.txt; exit 19 -n|--newconf) CONF_TEST=$2 ; shift 2;; -r|--refconf) CONF_REFE=$2 ; shift 2;;21 -h|--help ) cat trusting_help.txt; exit 1;; -m|--machine) CFG_ARCH=$2 ; shift 2;; 22 -n|--newconf) TEST_CONF=$2 ; shift 2;; -r|--refconf) REFE_CONF=$2 ; shift 2;; 20 23 -t|--time ) TIME_LIMI=$2 ; shift 2;; -p|--publish) PUBLISH=1 ; shift ;; 21 24 -u|--user ) CFG_USER=$2 ; shift 2;; -v|--version) NEMO_REVI=$2 ; shift 2;; 22 *) cat trusting_help.txt; exit 25 *) cat trusting_help.txt; exit 1;; 23 26 esac 24 27 done 25 28 26 # Set super-computer & user environment 27 [[ ! -z "${CFG_ARCH}" && ! -z "${CFG_USER}" ]] && . trusting.env || { ./trusting.sh -h && exit; } 28 export DEBUG PUBLISH 29 30 # Initialization 31 #============================================= 32 33 echo ${DIR_TEST} 34 mkdir -p ${DIR_TEST} ${DIR_REFE} 35 cd ${DIR_TEST} 36 datestr1=$( LC_ALL=C date -d"$DATE_STR" +%Y-%m-%dT%H:%M ) 37 printf "Date\n%s\n" $datestr1 > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt 38 39 printf "Status\nFAILED\n" > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt # Only at end status will change to OK 40 41 . trusting_func.sh 42 #============================================= 29 #export DEBUG PUBLISH 43 30 44 31 45 # 46 #============================================= 32 # Initialization (super-computer & user environment) 33 #--------------------------------------------------- 34 if [[ ! -z "${CFG_ARCH}" && ! -z "${CFG_USER}" ]]; then 35 36 if [ ! $( find ${NEMO_TRUS}/cfg -name ${CFG_USER}.cfg ) ]; then 37 echo 'No configuration file in '${NEMO_TRUS}'/cfg for '${CFG_USER}; exit 1 38 elif [ ! $( find ${NEMO_TRUS}/cfg -name ${CFG_ARCH}.cfg ) ]; then 39 echo 'No configuration file in '${NEMO_TRUS}'/cfg for '${CFG_ARCH}; exit 1 40 else 41 . ${NEMO_TRUS}/trusting.env 42 fi 43 44 else 45 cat ${NEMO_TRUS}/trusting_help.txt; exit 1 46 fi 47 48 datestr1=$( LC_ALL=C date -d"${DATE_STR}" +%Y-%m-%dT%H:%M ) 49 . ./trusting_func.sh 50 51 mkdir -p ${TEST_DIR} ${REFE_DIR} 52 echo ${TEST_DIR} && cd ${TEST_DIR} 53 printf "Date\n%s\n" $datestr1 > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt 54 # Only at end status will change to OK 55 printf "Status\nFAILED\n" > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt 56 57 58 # Check softwares versions 59 #--------------------------------------------------- 60 printf "Step......................\nGet softwares release\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 47 61 for rel in $CDO $COMPILER $MPI $NETCDF; do 48 printf "Step......................\nGet $rel release\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt49 62 arch_rel=$( echo $LOADEDMODULES | sed "s#.*:$rel/\([^:]*\).*#\1#" ) 50 63 [ -z "${arch_rel}" ] && arch_rel=$( echo $PATH | sed "s#.*$rel/\([^/]*\).*#\1#" ) 51 [ $( echo $rel | grep "$CDO\|$COMPILER" ) ] && arch_rel=$( $rel --version | grep -m1 -o ' [0-9.]* ')64 # [ $( echo $rel | grep $COMPILER ) ] && arch_rel=$( $rel --version | grep -m1 -o ' [0-9.]* ' >/dev/null ) 52 65 echo $rel ${arch_rel} >> model.log 53 66 done 54 67 55 sed -n 1p model.log | sed 'i\Fortran compiler' > mesg_04_compiler_${CFG_USER}_${CFG_ARCH}.txt56 sed -n 2p model.log | sed 'i\MPI libraries ' > mesg_05_mpi_${CFG_USER}_${CFG_ARCH}.txt57 sed -n 3p model.log | sed 'i\NetCDF libraries' > mesg_06_netcdf_${CFG_USER}_${CFG_ARCH}.txt68 sed -n 2p model.log | sed 'i\Fortran compiler' > mesg_04_compiler_${CFG_USER}_${CFG_ARCH}.txt 69 sed -n 3p model.log | sed 'i\MPI libraries ' > mesg_05_mpi_${CFG_USER}_${CFG_ARCH}.txt 70 sed -n 4p model.log | sed 'i\NetCDF libraries' > mesg_06_netcdf_${CFG_USER}_${CFG_ARCH}.txt 58 71 59 72 60 # ---------------------------------------------61 printf "Step......................\nCheckout XIOS, FCM-make(/EXTERNAL/fcm), NEMO engines (/NEMO) & configuration (/CONFIG)\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 73 # SVN action on XIOS & NEMO essentials directories 74 #--------------------------------------------------- 62 75 svn_cmd='svn status' 63 64 76 # If -v|--version option has been set, modify svn command 65 77 [ $( echo ${NEMO_VERS} | grep "up\|update" ) ] && svn_cmd='svn update' 66 78 [ $( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) ] && svn_cmd='svn update -r '$( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) 67 79 80 printf "Step......................\nGet NEMO SVN revision with: ${svn_cmd}\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 68 81 rev=0 82 69 83 for dir in ${DIR_XIOS} ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_EFCM} ${NEMO_ENGI} ${NEMO_IPSL} ${NEMO_TRUS} ${NEMO_TCMP}; do 84 # For time being, just get revision number from XIOS 70 85 [ $dir == ${DIR_XIOS} ] && echo 'XIOS '$( svn info $dir | awk '(NR == 9) {print $NF}' ) >> model.log && continue 71 86 ${svn_cmd} $dir … … 76 91 77 92 78 #--------------------------------------------- 79 printf "Step......................\nCompilation NEMO configuration %s\n" ${CONF_REF} > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 93 # NEMO config compiled from scratch 94 #--------------------------------------------------- 95 printf "Step......................\nCompiling ${CONF_REF} configuration\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 80 96 cd ${NEMO_CONF} 81 . ${ARCH_ENV} # Sourcing architecture environment82 #[ -d ${ CONF_TEST} ] && ./makenemo -n ${CONF_TEST} clean83 ./makenemo -n ${ CONF_TEST} -r ${CONF_REFE} -m $ARCH -j $NPROC #> /dev/null 2>&184 [ ! -e ${ CONF_TEST}/BLD/bin/nemo.exe ] && getout97 . ${ARCH_ENV} > /dev/null # Sourcing architecture environment 98 #[ -d ${TEST_CONF} ] && ./makenemo -n ${TEST_CONF} clean 99 ./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m $ARCH -j $NPROC > /dev/null 2>&1 100 [ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out 85 101 86 102 87 #--------------------------------------------- 88 printf "Step......................\nPreparing job\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 89 cd ${DIR_TEST} 90 find ${NEMO_CONF}/${CONF_TEST}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \; 103 # Get namelists, xml & forced files for running 104 #--------------------------------------------------- 105 printf "Step......................\nPreparing job\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 106 cd ${TEST_DIR} 107 find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \; 91 108 92 109 if [ ! -z "${NEMO_FORC}" ]; then 93 110 cp ${NEMO_FORC}/${FORC_TAR} . 94 tar -tvf ${FORC_TAR} > inputs_list.txt && tar -vxf ${FORC_TAR} 95 [ $? -ne 0 ] && get out111 tar -tvf ${FORC_TAR} > inputs_list.txt && tar -vxf ${FORC_TAR} > /dev/null 112 [ $? -ne 0 ] && get_out 96 113 gunzip *.gz 97 114 else … … 100 117 101 118 102 # ---------------------------------------------103 printf "Step......................\nRunning\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 104 #${JOB_SUB} ${DIR_TEST}/../../batch.sh 105 job_num=$( ${JOB_SUB } ${DIR_TEST}/../../batch.sh | awk '{print $NF}')106 [ $? -ne 0 ] && get out119 # Job submission & computing 120 #--------------------------------------------------- 121 printf "Step......................\nRunning\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt; echo ${JOB_SUBM} 122 job_num=$( ${JOB_SUBM} | awk '{print $NF}' ) 123 [ $? -ne 0 ] && get_out 107 124 108 time_elapsed=0; time_increment=30 0125 time_elapsed=0; time_increment=30 109 126 sleep ${time_increment} 110 while [[ $( ${JOB_LIST} | grep $job_num ) && ${time_elapsed} -lt ${TIME_LIMI} ]] 127 while [[ $( ${JOB_LIST} | grep $job_num ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do 111 128 printf "\n####################################################" >> computation.log 112 129 ${JOB_VIEW} $job_num >> computation.log … … 116 133 117 134 118 #--------------------------------------------- 119 printf "Step......................\nTest if ASCII output files exist\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 120 if [ ! -f ${output_test} ]; then 121 122 getout 123 else if [ ] 124 125 grep 'E R R O R' ocean.output >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt 126 sed 's/^/Time step :/' time.step >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt 135 # Eventual crash during computing 136 #--------------------------------------------------- 137 printf "Step......................\nTest if ASCII output files exist\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 138 if [ ! -f ocean.output ]; then 139 printf "Step......................\nOpa crashed at initialization\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 140 status='FAILED' 141 elif [ $( grep 'E R R O R' ocean.output ) ]; then 142 printf "Step......................\nOpa crashed\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 143 grep 'E R R O R' ocean.output >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt 144 sed 's/^/Time step :/' time.step >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt 145 status='FAILED' 127 146 fi 128 147 129 148 130 #--------------------------------------------- 131 printf "Step......................\nTest ASCII output files difference\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 149 # Inspect output text files 150 #--------------------------------------------------- 151 printf "Step......................\nTest ASCII output files difference\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 132 152 for file in ocean.output solver.stat tracer.stat; do 133 diff $file ${ DIR_REFE}/$file153 diff $file ${REFE_DIR}/$file 134 154 [ $? -ne 0 ] && status='FAILED' 135 155 done 136 156 137 157 138 #--------------------------------------------- 139 printf "Step......................\nTest last restart files difference\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 140 diff_trusting 2>/dev/null 158 # Inspect output NetCDF files 159 #--------------------------------------------------- 160 printf "Step......................\nTest last restart NetCDF files difference\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 161 diff_restart 2>/dev/null 141 162 [ $? -ne 0 ] && status='FAILED' 142 163 143 164 144 #--------------------------------------------- 165 # Comments 166 #--------------------------------------------------- 145 167 echo 'Comments' > mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt 146 168 grep 'W A R N I N G:' ocean.output >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt 147 169 148 170 149 #--------------------------------------------- 171 # Finals tests 172 #--------------------------------------------------- 150 173 if [ $status != 'FAILED' ]; then 151 printf "Step......................\nGet Real CPU time\n" >mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt152 real_cpu_time=$( ${JOB_VIEW} $job_num | awk 'END{print $9}' )174 printf "Step......................\nGet real CPU time\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 175 real_cpu_time=$( ${JOB_VIEW} $job_num | awk 'END{print $9}' ) 153 176 printf "Real CPU time\n%s\n" ${real_cpu_time} > mesg_08_realcputime_${CFG_USER}_${CFG_ARCH}.txt 154 177 else 155 printf "Step......................\nTest InputFiles difference\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 156 diff inputs_list.txt ${DIR_REFE}/inputs_list.txt 157 [ $? -ne 0 ] && getout 158 [[ $? -ne 0 && printf "InputFiles\n%s\n" "Different" > mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt ]] || printf "InputFiles\n%s\n" "Same" > mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt 178 printf "Step......................\nTest input files difference\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 179 diff inputs_list.txt ${REFE_DIR}/inputs_list.txt 180 181 if [ $? -ne 0]; then 182 printf "Input files\nDifferent" > mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt 183 else 184 printf "Input files\nSame" > mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt 185 fi 186 187 get_out 159 188 fi 160 189 161 190 162 #--------------------------------------------- 191 # So far, everything is good 192 #--------------------------------------------------- 163 193 printf "Status\nOK\n" > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt 164 printf "Step......................\nCode is reliable\n" >mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt194 printf "Step......................\nCode is reliable\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt 165 195 166 #--------------------------------------------- 167 getout 196 197 # End 198 #--------------------------------------------------- 199 get_out
Note: See TracChangeset
for help on using the changeset viewer.