Changeset 5644 for branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST
- Timestamp:
- 2015-07-29T20:33:51+02:00 (9 years ago)
- Location:
- branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST
- Files:
-
- 2 added
- 8 edited
- 1 copied
- 1 moved
Legend:
- Unmodified
- Added
- Removed
-
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/config/X64_ADA.cfg
r5636 r5644 8 8 9 9 # Job commands: LoadLeveler 10 JOB_SUBM="llsubmit ${NEMO_TRUS}/batch/${ CFG_ARCH}.sh | grep -oe '\<[0-9]*\>'"10 JOB_SUBM="llsubmit ${NEMO_TRUS}/batch/${NEMO_HPCC}.sh | grep -oe '\<[0-9]*\>'" 11 11 JOB_LIST=llq 12 12 JOB_INFO='llq -x' … … 19 19 20 20 # Architecture environment (be careful with compliance with XIOS) 21 ARCH_ENV=${DIR_XIOS}/arch/arch-${ CFG_ARCH}.env21 ARCH_ENV=${DIR_XIOS}/arch/arch-${NEMO_HPCC}.env -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/config/X64_CURIE.cfg
r5635 r5644 3 3 # Available softwares: 4 4 CDO='cdo' 5 COMPILER='i ntel'5 COMPILER='ifort' 6 6 MPI='bullxmpi' 7 7 NETCDF='netcdf' 8 8 9 9 # Job commands: ccc_* 10 JOB_SUBM="ccc_msub ${NEMO_TRUS}/batch/${ CFG_ARCH}.sh | grep -oe '\<[0-9]*\>'"10 JOB_SUBM="ccc_msub ${NEMO_TRUS}/batch/${NEMO_HPCC}.sh | grep -oe '\<[0-9]*\>'" 11 11 JOB_LIST=ccc_mpp 12 12 JOB_INFO=ccc_macct … … 19 19 20 20 # Architecture environment (be careful with compliance with XIOS) 21 ARCH_ENV=${DIR_XIOS}/arch/arch-${ CFG_ARCH}.env21 ARCH_ENV=${DIR_XIOS}/arch/arch-${NEMO_HPCC}.env -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/config/arch_template.cfg
r5483 r5644 1 # Copy to ./${ CFG_ARCH}.cfg, fill in according to your installation1 # Copy to ./${NEMO_HPCC}.cfg, fill in according to your installation 2 2 3 3 # Super-computer configuration … … 15 15 # Job commands (works with job ID): 16 16 JOB_SUBM='' # Submit a job && get his ID in return 17 # ex: "cmd ${NEMO_TRUS}/batch/${ CFG_ARCH}.sh | awk '{print \$NF}'"17 # ex: "cmd ${NEMO_TRUS}/batch/${NEMO_HPCC}.sh | awk '{print \$NF}'" 18 18 JOB_LIST='' # List all submitted jobs (to determine by `grep` if a job completed or not) 19 19 JOB_INFO='' # Get job informations with job ID as argument (filling a computation log) … … 27 27 28 28 # Architecture environment (be careful with compliance with XIOS) 29 ARCH_ENV='' # ${DIR_WORK}/XIOS/arch/arch-${ CFG_ARCH}.env29 ARCH_ENV='' # ${DIR_WORK}/XIOS/arch/arch-${NEMO_HPCC}.env -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/config/user_template.cfg
r5627 r5644 7 7 [ -z "${CONF_REFE}" ] && REFE_CONF='' # ex: 'ORCA2_LIM_PISCES', 'GYRE', ... 8 8 [ -z "${CONF_TEST}" ] && TEST_CONF='' # ex: "${CONF_REFE}_trust", ... 9 [ -z "${ FORC_TARF}" ] && FORC_TARF='' # Archive name (ex: 'ORCA2_LIM_nemo_v3.6.tar', ...), let blank if not requested9 [ -z "${NEMO_TARF}" ] && NEMO_TARF='' # Archive name (ex: 'ORCA2_LIM_nemo_v3.6.tar', ...), let blank if not requested 10 10 NEMO_HOME='' # NEMOGCM directory for computation (./ARCH, ./CONFIG, ...) 11 11 # ex: ".../NEMO/${NEMO_BRAN}/NEMOGCM", ... … … 17 17 # ex: ".../${CONF_REFE}/${NEMO_BRAN}" 18 18 TEST_DIR='' # Test folder for computation 19 # ex: ".../${CONF_REFE}/${NEMO_BRAN}/trusting_${DATE _STR}"19 # ex: ".../${CONF_REFE}/${NEMO_BRAN}/trusting_${DATE}" 20 20 21 21 # External softs directories: -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/install_dir.sh
r5542 r5644 3 3 #settings 4 4 /ccc/work/cont003/gen7451/martin/NEMO/ 5 branches/2015/nemo_v3_6_STABLE5 NEMO_BRAN=branches/2015/nemo_v3_6_STABLE 6 6 7 X64_CURIE_ trust7 X64_CURIE_cron 8 8 9 9 cd /ccc/work/cont003/gen7451/martin/NEMO 10 10 mkdir -p ${}/NEMOGCM 11 11 12 for elmt in ARCH CONFIG NEMO EXTERNAL/IOIPSL EXTERNAL/fcm TOOLS ; do 13 svn co http://forge.ipsl.jussieu.fr/nemo/svn/${}/NEMOGCM/$elmt ${}/NEMOGCM/$elmt 12 for elmt in 'ARCH' 'CONFIG' 'NEMO' 'EXTERNAL/IOIPSL' 'EXTERNAL/fcm' 'TOOLS' 'TRUST'; do 13 svn_bran=${NEMO_BRAN} 14 [ "$elmt" == 'TRUST' ] && svn_bran='branches/2015/dev_r5092_CNRS_SETTE' 15 svn co http://forge.ipsl.jussieu.fr/nemo/svn/${svn_bran}/NEMOGCM/$elmt ${}/NEMOGCM/$elmt 16 14 17 done 15 18 16 19 cd NEMOGCM/TOOLS 17 cp -f ../../../trunk/NEMOGCM/ARCH/CNRS/arch-X64_CURIE_trust.fcm ../ARCH18 ./maketools -m ${} -n REBUILD_NEMO > /dev/null19 20 20 21 for elmt in $( ls * ); do 21 22 [ ! $( echo $elmt | grep "COMPILE\|maketools\|REBUILD_NEMO" ) ] && rm -rf $elmt 22 23 done 24 25 if [ find ../ARCH -name ] 26 ./maketools -m ${} -n REBUILD_NEMO > /dev/null 27 -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.env
r5629 r5644 2 2 3 3 4 ## Time-stamp 5 date_test=$( date ) 6 export DATESTR=$( date -ud "${date_test}" +%Y%m%d%Z%H%M ) 4 ## Time-stamp & filename pattern 5 dat=$( date ) 6 export DATE=$( date -ud "$dat" +"%Y%m%d%Z%H%M" ) 7 export PATTERNAME=${NEMO_USER}'_'${NEMO_HPCC} 7 8 8 9 ## Sourcing user configuration 9 10 ##-------------------------------------- 10 . ${NEMO_TRUS}/config/${ CFG_USER}.cfg11 . ${NEMO_TRUS}/config/${NEMO_USER}.cfg 11 12 export NEMO_HOME NEMO_BRAN 12 13 export REFE_CONF TEST_CONF REFE_DIR TEST_DIR 13 14 export DIR_XIOS #DIR_OASI 14 export NEMO_FORC FORC_TARF15 export NEMO_FORC NEMO_TARF 15 16 export EMAIL 16 17 … … 25 26 ## Sourcing super-computer configuration 26 27 ##-------------------------------------- 27 . ${NEMO_TRUS}/config/${ CFG_ARCH}.cfg28 . ${NEMO_TRUS}/config/${NEMO_HPCC}.cfg 28 29 export CDO COMPILER MPI NETCDF 29 30 export JOB_SUBM JOB_LIST JOB_INFO JOB_TIME JOB_DELE -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh
r5637 r5644 2 2 3 3 4 cd $( dirname $0 ) 5 4 6 ## Set defaults 5 7 ##--------------------------------------------------- 6 NEMO_TRUS=$( pwd $( dirname $0 ))8 NEMO_TRUS=$( pwd ) 7 9 DEBUG=0; PUBLISH=0 8 10 I_MODULE=0 ## No 'module' command by default … … 13 15 while [ $# -ne 0 ]; do 14 16 case $1 in 15 -a|--archive) FORC_TARF=$2 ; shift 2;; -b|--branch ) NEMO_BRAN=$2 ; shift 2;;17 -a|--archive) NEMO_TARF=$2 ; shift 2;; -b|--branch ) NEMO_BRAN=$2 ; shift 2;; 16 18 -d|--debug ) set -vx; DEBUG=1 ; shift ;; -e|--email ) EMAIL=$2 ; shift 2;; 17 19 -j|--job ) NPROC=$2 ; shift 2;; -h|--help ) cat trusting_help.txt; exit 1;; 18 -m|--machine) CFG_ARCH=$2; shift 2;; -n|--newconf) TEST_CONF=$2 ; shift 2;;20 -m|--machine) NEMO_HPCC=$2 ; shift 2;; -n|--newconf) TEST_CONF=$2 ; shift 2;; 19 21 -r|--refconf) REFE_CONF=$2 ; shift 2;; -t|--time ) TIME_LIMI=$2 ; shift 2;; 20 -p|--publish) PUBLISH=1 ; shift ;; -u|--user ) CFG_USER=$2; shift 2;;22 -p|--publish) PUBLISH=1 ; shift ;; -u|--user ) NEMO_USER=$2 ; shift 2;; 21 23 -v|--version) NEMO_VERS=$2 ; shift 2;; *) cat trusting_help.txt; exit 1;; 22 24 esac … … 25 27 ## Initialization (super-computer & user environment) 26 28 ##--------------------------------------------------- 27 if [[ ! -z "${ CFG_ARCH}" && ! -z "${CFG_USER}" ]]; then29 if [[ ! -z "${NEMO_HPCC}" && ! -z "${NEMO_USER}" ]]; then 28 30 29 if [ ! $( find ${NEMO_TRUS}/config -name ${ CFG_USER}.cfg) ]; then30 echo 'No configuration file in '${NEMO_TRUS}'/config for '${CFG_USER}; exit 131 elif [ ! $( find ${NEMO_TRUS}/config -name ${ CFG_ARCH}.cfg) ]; then32 echo 'No configuration file in '${NEMO_TRUS}'/config for '${CFG_ARCH}; exit 131 if [ ! $( find ${NEMO_TRUS}/config -name ${NEMO_USER}'.cfg' ) ]; then 32 echo 'No configuration file (.cfg) in '${NEMO_TRUS}'/config for '${NEMO_USER}; exit 1 33 elif [ ! $( find ${NEMO_TRUS}/config -name ${NEMO_HPCC}'.cfg' ) ]; then 34 echo 'No configuration file (.cfg) in '${NEMO_TRUS}'/config for '${NEMO_HPCC}; exit 1 33 35 else 34 36 . ${NEMO_TRUS}/trusting.env … … 41 43 . ./trusting_func.sh 42 44 43 ## 45 ## Summary of trusting test context 44 46 ##--------------------------------------------------- 47 printf "\ 48 NEMO Trusting test:\n\ 49 -------------------\n\ 50 Testing ${TEST_CONF} based on ${REFE_CONF} from ${NEMO_BRAN}\n\ 51 Configuration files: ${NEMO_USER}.cfg & ${NEMO_HPCC}.cfg\n" 52 53 ## Making timestamped directory with messenger files 54 ##--------------------------------------------------- 55 print_step 'Time-stamp generation & testing directory creation' 45 56 mkdir -p ${TEST_DIR} ${REFE_DIR} 46 57 echo ${TEST_DIR} && cd ${TEST_DIR} 47 date_str=$( date -ud ${DATESTR} +"%F %R %Z" ) 48 printf "Date\n${date_str}\n" > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt 58 init_files && get_date 49 59 50 60 ## SVN action on XIOS & NEMO essentials directories 51 61 ##--------------------------------------------------- 52 print_step 'SVN actions on NEMO directories'62 print_step 'SVN command on NEMO directories:' 53 63 get_nemo_rev 54 64 55 65 ## Check softwares versions (source arch environment) 56 66 ##--------------------------------------------------- 57 print_step 'Get softwares release '67 print_step 'Get softwares releases' 58 68 [ -e ${ARCH_ENV} ] && . ${ARCH_ENV} > /dev/null 59 [ $? -eq 0 ] && I_MODULE=1 60 export I_MODULE 69 (( $? == 0 )) && export I_MODULE=1 61 70 get_soft_rel 62 71 cat model.log … … 64 73 ## XIOS compiling 65 74 ##--------------------------------------------------- 66 print_step 'Compil ingXIOS'75 print_step 'Compile XIOS' 67 76 cd ${DIR_XIOS} 68 77 full=''; [ $DEBUG -eq 0 ] && full='--full' 69 ./make_xios $full --arch ${ CFG_ARCH} --job $NPROC >& /dev/null78 ./make_xios $full --arch ${NEMO_HPCC} --job $NPROC >& /dev/null 70 79 [ ! -e ${DIR_XIOS}/lib/libxios.a ] && get_out 1 71 80 72 81 ## NEMO config compiled from scratch 73 82 ##--------------------------------------------------- 74 print_step "Compil ing${REFE_CONF} configuration"83 print_step "Compile ${REFE_CONF} configuration" 75 84 cd ${NEMO_CONF} 76 85 [[ -d ${TEST_CONF} && $DEBUG -eq 0 ]] && ./makenemo -n ${TEST_CONF} clean > /dev/null 77 ./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m ${ CFG_ARCH} -j $NPROC>& /dev/null86 ./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m ${NEMO_HPCC} -j $NPROC >& /dev/null 78 87 [ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out 2 79 88 80 89 ## Get namelists, xml & forcing files for running 81 90 ##--------------------------------------------------- 82 print_step "Preparing job by copying ${REFE_CONF} input files"91 print_step 'Set job (copying & extracting input files)' 83 92 cd ${TEST_DIR} 84 93 find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \; 85 94 get_inputs 86 [ $? -ne 0 ]&& get_out 395 (( $? != 0 )) && get_out 3 87 96 [ $( find . -name '*.gz' -print -quit ) ] && gunzip *.gz 88 97 89 98 ## Check input files in all cases 90 99 ##--------------------------------------------------- 91 print_step ' Difference with standard input files list'100 print_step 'Compare with standard input files list' 92 101 diff_inputs 93 102 94 103 ## Job submission & computing 95 104 ##--------------------------------------------------- 96 print_step 'Submit tingjob'105 print_step 'Submit job' 97 106 JOB_ID=$( eval ${JOB_SUBM} ) 98 [ $? -ne 0 ]&& get_out 499 print_step ' Holding/Running job'107 (( $? != 0 )) && get_out 4 108 print_step 'Pending job' 100 109 job_pending 101 110 print_step 'Job finished' -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh
r5638 r5644 2 2 3 3 4 comments() { 5 state=$1; LAST_TIME_STEP=0 6 7 if [ "$state" == 'E R R O R' ]; then 8 [ -e time.step ] && LAST_TIME_STEP=$( cat time.step | tr -d ' ' ) 9 export LAST_TIME_STEP 10 [ -e ocean.output ] && line=$( grep -m1 -A4 "$state" ocean.output | tr -d '\n' ) 11 else 12 [ -e ocean.output ] && line=$( grep -m1 -A2 "^ $state" ocean.output | tr -d '\n' ) 13 fi 14 15 # if [[ -e ${REFE_DIR}/model.log && $( diff -q model.log ${REFE_DIR}/model.log ) ]]; then 16 # line=$( diff model.log ${REFE_DIR}/model.log | grep '>' | sed '{N;s/\n/<BR>/}' ) 17 # fi 18 19 printf "Comments\n$line\n" | tee mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt 20 } 21 22 23 diff_inputs() { 24 mesg='Same' 25 for file in inputs_list.txt $( ls namelist_* ) $( ls *.xml ); do 26 diff -q $file ${REFE_DIR}/$file > /dev/null 27 [ $? -ne 0 ] && mesg='Different' 28 done 29 30 printf "Input files\n$mesg\n" | tee mesg_09_inputs_${CFG_USER}_${CFG_ARCH}.txt 31 } 32 33 diff_results() { 34 for file in ocean.output $( ls *.stat ); do 35 [ ! -e ${REFE_DIR}/$file ] && export ST='FAILED' && get_out 7 36 diff -q $file ${REFE_DIR}/$file 37 [ $? -ne 0 ] && export ST='FAILED' 38 done 39 } 40 41 diff_restart() { 42 [ ! -e ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 9 43 44 export LAST_TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 45 echo 'Last time step of standard run: '${LAST_TIME_STEP} 46 47 if [ $( find -regex ".*_0+${LAST_TIME_STEP}_restart.*\.nc" -print -quit ) ]; then 48 base_name=$( find -regex ".*_0+${LAST_TIME_STEP}_restart.*\.nc" -print -quit | awk -F/ '{print $NF}' \ 49 | sed "s/^\(.*\)$LAST_TIME_STEP\_restart.*/\1$LAST_TIME_STEP\_/" ) 50 51 for comp in restart restart_ice restart_trc; do 52 file=${base_name}${comp} && printf "$file.nc: " 53 54 nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{print $1}' ) 55 if [ ${nb_dom} -gt 1 ]; then 56 ${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null 57 [ $? -eq 0 ] && rm -f ${base_name}${comp}_[0-9]*.nc > /dev/null 58 elif [ ${nb_dom} -eq 0 ]; then 59 export ST='FAILED' && get_out 10 60 fi 61 62 if [ -e ${REFE_DIR}/$file.nc ]; then 63 ## UNIX `cmp` not suitable (filename & timestamp in .nc file) 64 nc_diff=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 65 | awk '/records/ {print $1}' ) 66 67 if [[ ! -z ${nc_diff} && ${nc_diff} -ne 0 ]]; then 68 export ST='FAILED' 69 printf "$CDO ${nc_diff}\n" 70 else 71 printf "identical\n" 72 fi 73 74 else 75 export ST='FAILED' && get_out 9 76 fi 77 78 done 79 80 else 81 export ST='FAILED' 82 fi 83 84 } 85 86 get_cpu_time() { 87 real_cpu_time=$( eval ${JOB_TIME} ) 88 printf "Real CPU time\n${real_cpu_time}\n" | tee mesg_10_cputime_${CFG_USER}_${CFG_ARCH}.txt 89 } 90 91 get_inputs() { 92 if [ ! -z "${FORC_TARF}" ]; then 93 cmd_iol="tar -tvf ${NEMO_FORC}/${FORC_TARF}"; cmd_iof="tar -vxf ${NEMO_FORC}/${FORC_TARF}" 94 else 95 cmd_iol="ls ${NEMO_FORC}/*" ; cmd_iof="\cp ${NEMO_FORC}/* ." 96 fi 97 98 ${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null 99 } 100 101 get_soft_rel() { 102 for soft in $CDO $COMPILER $MPI $NETCDF; do 103 soft_rel='' 104 (( ${I_MODULE} == 1 )) && soft_rel=$( echo $LOADEDMODULES | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i" ) 105 [ -z "${soft_rel}" ] && soft_rel=$( echo $PATH | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i" ) 106 [ "$soft" == "$COMPILER" ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' ) 107 # Cleaning characters string to display proper soft name 108 soft=$( echo $soft | sed 's#\\##; s#[/-]$##' ) 109 echo $soft ${soft_rel} >> model.log 110 done 111 112 sed -n 4p model.log | sed 'i\Fortran compiler' > mesg_06_compiler_${CFG_USER}_${CFG_ARCH}.txt 113 sed -n 5p model.log | sed 'i\MPI libs' > mesg_07_mpi_${CFG_USER}_${CFG_ARCH}.txt 114 sed -n 6p model.log | sed 'i\NetCDF libs' > mesg_08_netcdf_${CFG_USER}_${CFG_ARCH}.txt 4 ## Messenger filenames 5 FILE_DATE=mesg_01_date_$PATTERNAME.txt ; FILE_STAT=mesg_02_status_$PATTERNAME.txt 6 FILE_RESU=mesg_03_result_$PATTERNAME.txt ; FILE_NEMO=mesg_04_nemogcm_$PATTERNAME.txt 7 FILE_XIOS=mesg_05_xios_$PATTERNAME.txt ; FILE_COMP=mesg_06_compiler_$PATTERNAME.txt 8 FILE_MPIN=mesg_07_mpi_$PATTERNAME.txt ; FILE_NCDF=mesg_08_netcdf_$PATTERNAME.txt 9 FILE_INPT=mesg_09_inputs_$PATTERNAME.txt ; FILE_CPUT=mesg_10_cputime_$PATTERNAME.txt 10 FILE_COMM=mesg_11_comments_$PATTERNAME.txt; 11 12 ## 13 FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tar.gz 14 15 print_step() { printf "\nStep.....\n$1\n"; } 16 17 init_files() { 18 echo 'Date' > ${FILE_DATE}; echo 'Status' > ${FILE_STAT} 19 echo 'Result' > ${FILE_RESU}; echo 'NEMOGCM rev.' > ${FILE_NEMO} 20 echo 'XIOS rev.' > ${FILE_XIOS}; echo 'Fortran compiler' > ${FILE_COMP} 21 echo 'MPI libs' > ${FILE_MPIN}; echo 'NetCDF libs' > ${FILE_NCDF} 22 echo 'Input files' > ${FILE_INPT}; echo 'Real CPU time' > ${FILE_CPUT} 23 echo 'Comments' > ${FILE_COMM} 24 } 25 26 get_date() { 27 local dat=$( date -ud ${DATE} +"%F %R %Z" ) 28 29 echo $dat >> ${FILE_DATE} 115 30 } 116 31 117 32 get_nemo_rev() { 118 svn_opt='status' 33 local rev_loc svn_cmd 34 local rev=0 svn_opt='status' 119 35 120 36 ## If -v|--version option has been set, modify svn command … … 129 45 fi 130 46 131 rev=0132 47 for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \ 133 48 ${NEMO_EIOI} ${NEMO_EFCM} \ … … 138 53 if [ "$dir" == "${DIR_XIOS}" ]; then 139 54 rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) 140 echo 'XIOS '${rev_loc} >> model.log 141 printf "XIOS rev.\n" > mesg_05_xios_${CFG_USER}_${CFG_ARCH}.txt 142 printf "<a href=\"https://forge.ipsl.jussieu.fr/ioserver/changeset/${rev_loc}\" target=\"_blank\">${rev_loc}</a>" \ 143 >> mesg_05_xios_${CFG_USER}_${CFG_ARCH}.txt 55 echo 'XIOS '${rev_loc} >> model.log 56 echo "<a href=\"https://forge.ipsl.jussieu.fr/ioserver/changeset/${rev_loc}\" target=\"_blank\">${rev_loc}</a>" \ 57 >> ${FILE_XIOS} 144 58 continue 145 59 fi 146 60 147 echo ${svn_cmd} $dir': '$( ${svn_cmd} $dir ) 61 echo $dir 62 ${svn_cmd} $dir 148 63 rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) 149 64 (( ${rev_loc} >= $rev )) && rev=${rev_loc} 150 65 done 151 66 152 echo 'NEMOGCM '$rev >> model.log 153 printf "NEMOGCM rev.\n" > mesg_04_nemogcm_${CFG_USER}_${CFG_ARCH}.txt 154 printf "<a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \ 155 >> mesg_04_nemogcm_${CFG_USER}_${CFG_ARCH}.txt 67 echo 'NEMOGCM '$rev >> model.log 68 echo "<a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \ 69 >> ${FILE_NEMO} 70 } 71 72 get_soft_rel() { 73 local soft_rel 74 75 for soft in $CDO $COMPILER $MPI $NETCDF; do 76 soft_rel='' 77 (( ${I_MODULE} == 1 )) && soft_rel=$( echo $LOADEDMODULES | sed "s/.*$soft\/\([0-9.a-z_]*\).*/\1/i" ) 78 [ -z "${soft_rel}" ] && soft_rel=$( echo $PATH | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i" ) 79 [ "$soft" == "$COMPILER" ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' ) 80 ## Cleaning characters string to display proper soft name 81 soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' ) 82 echo $soft ${soft_rel} >> model.log 83 done 84 85 sed -n 4p model.log > ${FILE_COMP}; sed -n 5p model.log > ${FILE_MPIN}; sed -n 6p model.log > ${FILE_NCDF} 86 } 87 88 get_inputs() { 89 local cmd_iol="tar -tvf ${NEMO_FORC}/${NEMO_TARF}"; cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}" 90 91 [ -z "${NEMO_TARF}" ] && cmd_iol="ls ${NEMO_FORC}/*" && cmd_iof="\cp ${NEMO_FORC}/* ." 92 ${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null 93 } 94 95 diff_inputs() { 96 local dif 97 local mesg='Same' 98 99 for file in inputs_list.txt $( ls namelist_* ) $( ls *.xml ); do 100 dif='' 101 if [ -e ${REFE_DIR}/$file ]; then dif=$( diff -q $file ${REFE_DIR}/$file ); else dif=0; fi 102 if [ -n "$dif" ]; then mesg='Different'; [ "$dif" != '0' ] && echo $dif | tee -a ${FILE_COMM}; fi 103 done 104 105 echo $mesg | tee -a ${FILE_INPT} 106 } 107 108 job_pending() { 109 local time_elapsed=0; time_increment=30 110 111 sleep ${time_increment} 112 113 while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do 114 printf "\n####################################################\n" >> computation.log 115 ${JOB_INFO} ${JOB_ID} >> computation.log 116 sleep ${time_increment} 117 time_elapsed=$(( ${time_elapsed} + ${time_increment} )) 118 done 119 120 sleep ${time_increment} 121 122 if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then 123 ${JOB_DELE} ${JOB_ID} &> /dev/null 124 TIME_LIMIT=$(( ${TIME_LIMI} / 3600 )) 125 # export TIME_LIMIT 126 get_out 6 127 fi 128 } 129 130 diff_results() { 131 for file in ocean.output $( ls *.stat ); do 132 [ ! -e ${REFE_DIR}/$file ] && export ST='FAILED' && get_out 7 133 diff -q $file ${REFE_DIR}/$file | tee -a ${FILE_COMM} 134 (( $? != 0 )) && export ST='FAILED' 135 done 136 } 137 138 diff_restart() { 139 local base_name file nb_dom diff_rcd 140 141 [ ! -e ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 9 142 143 # export LAST_TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 144 LAST_TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 145 echo 'Last time step of standard run: '${LAST_TIME_STEP} 146 147 if [ $( find -regex ".*_0+${LAST_TIME_STEP}_restart.*\.nc" -print -quit ) ]; then 148 base_name=$( find -regex ".*_0+${LAST_TIME_STEP}_restart.*\.nc" -print -quit | awk -F/ '{print $NF}' \ 149 | sed "s/^\(.*\)$LAST_TIME_STEP\_restart.*/\1$LAST_TIME_STEP\_/" ) 150 151 for comp in restart restart_ice restart_trc; do 152 file=${base_name}${comp} && printf "$file.nc: " 153 154 nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{print $1}' ) 155 if [ ${nb_dom} -gt 1 ]; then 156 ${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null 157 [ $? -eq 0 ] && rm -f ${base_name}${comp}_[0-9]*.nc > /dev/null 158 elif [ ${nb_dom} -eq 0 ]; then 159 export ST='FAILED' && get_out 10 160 fi 161 162 if [ -e ${REFE_DIR}/$file.nc ]; then 163 ## UNIX `cmp` not suitable (filename & timestamp in .nc file) 164 # diff_rcd=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 165 # | awk '/records/ {print $1}' ) 166 diff_rcd=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 167 | awk '/records/ {print $0}' ) 168 169 # if [[ ! -z ${diff_rcd} && ${diff_rcd} -ne 0 ]]; then 170 if [[ ! -z ${diff_rcd} && $( echo ${diff_rcd} | awk '{print $1}') -ne 0 ]]; then 171 export ST='FAILED' 172 echo $CDO $( echo ${diff_rcd} | sed 's/differ$/differ,/' | tr -d '\n' ) 173 # printf "$CDO ${diff_rcd}\n" 174 else 175 echo 'identical' 176 # printf "identical\n" 177 fi 178 179 else 180 export ST='FAILED' && get_out 9 181 fi 182 183 done 184 185 else 186 export ST='FAILED' 187 fi 188 189 } 190 191 get_cpu_time() { 192 local real_cpu_time=$( eval ${JOB_TIME} ) 193 194 echo ${real_cpu_time} | tee -a ${FILE_CPUT} 195 } 196 197 comments() { 198 local state=$1 199 200 if [ "$state" == 'E R R O R' ]; then 201 [ -e time.step ] && LAST_TIME_STEP=$( cat time.step | tr -d ' ' ) 202 # export LAST_TIME_STEP 203 [ -e ocean.output ] && line=$( grep -m1 -A4 "$state" ocean.output | tr -d '\n' ) 204 echo $line >> ${FILE_COMM} 205 else 206 [ -e ocean.output ] && line=$( grep -m1 -A2 "^ $state" ocean.output | tr -d '\n' ) 207 fi 208 209 echo $line 156 210 } 157 211 158 212 mesg_make() { 159 paste -d ';' mesg_* _${CFG_USER}_${CFG_ARCH}.txt > trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt160 cat trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt213 paste -d ';' mesg_*.txt > ${FILE_TRUS} 214 cat ${FILE_TRUS} 161 215 } 162 216 163 217 mesg_publish() { 164 165 if [ $PUBLISH -eq 1 ]; then 166 167 if [ -f ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt ]; then cmd='tail -1'; else cmd='cat'; fi 168 169 $cmd trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt >> ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt 218 local cmd 219 220 if [ $PUBLISH -eq 1 ]; then 221 222 if [ -f ${REFE_DIR}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi 223 224 $cmd ${FILE_TRUS} >> ${REFE_DIR}/trusting_$PATTERNAME.txt 170 225 171 226 ## Send mail only when FAILED 172 173 227 if [[ ! -z $EMAIL && "$ST" == 'FAILED' ]]; then 228 cat << END_MAIL > trusting.mail 174 229 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 175 230 … … 178 233 The trusting sequence for ${REFE_CONF} has failed. 179 234 Directory: ${TEST_DIR} 180 Archive created: trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.tar.gz235 Archive created: ${FILE_ARCH} in ${REFE_DIR} 181 236 182 237 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 183 238 184 239 END_MAIL 185 186 if [ -e ${TEST_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt ]; then 187 cat ${TEST_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt >> trusting.mail 188 fi 189 mail -s "[trusting ${REFE_CONF} $BRANCH] $ST $ERR" $EMAIL < trusting.mail 190 fi 191 192 fi 193 } 194 195 print_step() { printf "Step.....\n$1\n"; } 240 [ -e ${TEST_DIR}/${FILE_TRUS} ] && cat ${TEST_DIR}/${FILE_TRUS} >> trusting.mail 241 mail -s "[trusting ${REFE_CONF} $BRANCH] $ST $ERR" $EMAIL < trusting.mail 242 fi 243 244 fi 245 } 196 246 197 247 get_out() { … … 199 249 200 250 cd ${TEST_DIR} 201 printf "Status\n$ST\n" > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt251 echo $ST >> ${FILE_STAT} 202 252 203 253 if [ "$ST" == 'FAILED' ]; then 204 file=mesg_09_inputs_${CFG_USER}_${CFG_ARCH}.txt205 [ ! -e $file ] && printf "Input files\n\n" > $file206 file=mesg_10_cputime_${CFG_USER}_${CFG_ARCH}.txt207 [ ! -e $file ] && printf "Real CPU time\n\n" > $file208 file=mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt209 [ ! -e $file ] && printf "Comments\n\n" > $file210 254 211 255 ## Error identification … … 227 271 fi 228 272 229 printf "Result\n$ERR\n" > mesg_03_result_${CFG_USER}_${CFG_ARCH}.txt273 echo $ERR >> ${FILE_RESU} 230 274 231 275 ## Save tested configuration if trusting failed 232 276 if [[ "$ST" == 'FAILED' && $DEBUG -eq 0 ]]; then 233 echo 'Creating archive ' trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.tar.gz' under '${REFE_DIR}234 tar -czf ${REFE_DIR}/ trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.tar.gz*277 echo 'Creating archive '${FILE_ARCH}' under '${REFE_DIR} 278 tar -czf ${REFE_DIR}/${FILE_ARCH} ../* 235 279 fi 236 280 … … 240 284 exit 0 241 285 } 242 243 job_pending() {244 time_elapsed=0; time_increment=30245 246 sleep ${time_increment}247 while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do248 printf "\n####################################################\n" >> computation.log249 ${JOB_INFO} ${JOB_ID} >> computation.log250 sleep ${time_increment}251 time_elapsed=$(( ${time_elapsed} + ${time_increment} ))252 done253 sleep ${time_increment}254 255 if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then256 ${JOB_DELE} ${JOB_ID} &> /dev/null257 TIME_LIMIT=$(( ${TIME_LIMI} / 3600 ))258 export ${TIME_LIMIT}259 get_out $ST ERR=6260 fi261 }
Note: See TracChangeset
for help on using the changeset viewer.