Changeset 5788
- Timestamp:
- 2015-10-12T20:28:35+02:00 (9 years ago)
- Location:
- branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST
- Files:
-
- 3 added
- 4 deleted
- 13 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/batch/X64_ADA.sh
- Property svn:executable set to *
r5665 r5788 6 6 # @ job_type = parallel 7 7 # Fichier de sortie standard 8 # @ output = NEMO_CI9 # Fichier de sortie erreur (le meme)10 # @ error = NEMO_CI8 # @ output = $(job_name)_$(jobid) 9 # Fichier de sortie erreur 10 # @ error = $(job_name)_$(jobid) 11 11 # Nombre de processus demande 12 12 # @ total_tasks = 32 -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/batch/X64_CURIE.sh
r5644 r5788 3 3 #Config ccc_msub 4 4 #MSUB -A gen7451 5 #MSUB -r Trust6 #MSUB -o Trust_%I7 #MSUB -e Trust_%I5 #MSUB -r NEMO_CI 6 #MSUB -o NEMO_CI_%I 7 #MSUB -e NEMO_CI_%I 8 8 #MSUB -oe 9 9 #MSUB -q xlarge -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/config/X64_ADA.cfg
r5696 r5788 1 1 ## Ada configuration 2 ##------------------ 2 ##--------------------------------------------------- 3 3 4 ## Available softwares: 5 CDO='cdo' 6 COMPILER='ifort' 7 MPI='intel' 8 NETCDF='netcdf/mpi' 4 ARCH_ENV=${DIR_XIOS}/arch/arch-${TRUS_HPCC}.env 9 5 10 ## Job commands: LoadLeveler 11 JOB_SUBM="llsubmit ${NEMO_TRUS}/batch/${NEMO_HPCC}.sh | grep -oe '\<[0-9]*\>'" 12 JOB_LIST=llq 13 JOB_INFO='llq -x' 14 JOB_TIME="idrllsummary -j \${JOB_ID} | awk '/Step User Time/ {print \$4}' | cut -d. -f1" 15 JOB_DELE=llcancel 6 CMPF='ifort' 7 STR_CDOD='cdo/' 8 STR_LMPI='intel/impi/' 9 STR_NCDF='netcdf/' 16 10 17 ## Compiling & computing settings 18 [ -z "$NPROC" ] && NPROC=16 19 [ -z "${TIME_LIMI}" ] && TIME_LIMI=10800 # 3h 11 JOB_SUBM="llsubmit ${TRUS_DIR}/batch/${TRUS_HPCC}.sh | grep -oe '\<[0-9]*\>'" 12 JOB_STAT="llq | grep \${JOB_ID}" 13 JOB_INFO="llq -x \${JOB_ID}" 14 JOB_KILL="llcancel \${JOB_ID}" 20 15 21 ## Architecture environment (be careful with compliance with XIOS) 22 ARCH_ENV=${DIR_XIOS}/arch/arch-${NEMO_HPCC}.env 16 [ -z "${TRUS_NPRO}" ] && TRUS_NPRO='16' 17 [ -z "$TIMEOUT" ] && TIMEOUT='10800' 23 18 24 ## $CDO command to compare 2 datasets 19 JOB_TIME="grep IdrisMemMPI NEMO_CI_\${JOB_ID} | cut -d, -f1 | awk '{print \$(NF-1),\$NF}'" 20 JOB_PMEM="grep IdrisMemMPI NEMO_CI_\${JOB_ID} | cut -d, -f3 | awk '{print \$3,\$4}'" 21 JOB_VMEM="grep IdrisMemMPI NEMO_CI_\${JOB_ID} | cut -d, -f2 | awk '{print \$3,\$4}'" 22 25 23 CDOD='cdo diffn' -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/config/X64_CURIE.cfg
r5697 r5788 1 1 ## Curie configuration 2 ##-------------------- 2 ##--------------------------------------------------- 3 3 4 ## Available softwares: 5 CDO='cdo' 6 COMPILER='ifort' 7 MPI='bullxmpi' 8 NETCDF='netcdf' 4 ARCH_ENV=${DIR_XIOS}/arch/arch-${TRUS_HPCC}.env 9 5 10 ## Job commands: ccc_* 11 JOB_SUBM="ccc_msub ${NEMO_TRUS}/batch/${NEMO_HPCC}.sh | grep -oe '\<[0-9]*\>'" 12 JOB_LIST=ccc_mpp 13 JOB_INFO=ccc_macct 6 CMPF='ifort' 7 STR_CDOD='netcdf-utils-' 8 STR_LMPI='bullxmpi/' 9 STR_NCDF='netcdf-' 10 11 JOB_SUBM="ccc_msub ${TRUS_DIR}/batch/${TRUS_HPCC}.sh | grep -oe '\<[0-9]*\>'" 12 JOB_STAT="ccc_mpp | grep \${JOB_ID}" 13 JOB_INFO="ccc_macct \${JOB_ID}" 14 JOB_KILL="ccc_mdel \${JOB_ID}" 15 16 [ -z "${TRUS_NPRO}" ] && TRUS_NPRO='16' 17 [ -z "$TIMEOUT" ] && TIMEOUT='10800' 18 14 19 JOB_TIME="ccc_macct \${JOB_ID} | awk '/opa/ {print \$9}'" 15 JOB_DELE=ccc_mdel 20 JOB_PMEM="ccc_macct \${JOB_ID} | awk " 21 JOB_VMEM="ccc_macct \${JOB_ID} | awk " 16 22 17 ## Compiling & computing settings18 [ -z "$NPROC" ] && NPROC=1619 [ -z "${TIME_LIMI}" ] && TIME_LIMI=10800 # 3h20 21 ## Architecture environment (be careful with compliance with XIOS)22 ARCH_ENV=${DIR_XIOS}/arch/arch-${NEMO_HPCC}.env23 24 ## $CDO command to compare 2 datasets25 23 CDOD='cdo diffn' -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/config/X64_CURIE_cron.cfg
r5697 r5788 1 ## Curie configuration 2 ##-------------------- 1 ## Curie configuration (`cron` version) 2 ##--------------------------------------------------- 3 3 4 ## Available softwares: 5 CDO='netcdf-utils-' 6 COMPILER='ifort' 7 MPI='bullxmpi/' 8 NETCDF='netcdf-' 4 ARCH_ENV=${DIR_XIOS}/arch/arch-${TRUS_HPCC}.env 9 5 10 ## Job commands: ccc_* 11 JOB_SUBM="ccc_msub ${NEMO_TRUS}/batch/${NEMO_HPCC}.sh | grep -oe '\<[0-9]*\>'" 12 JOB_LIST=ccc_mpp 6 STR_CDOD='netcdf-utils-' 7 CMPF='ifort' 8 STR_LMPI='bullxmpi/' 9 STR_NCDF='netcdf-' 10 11 JOB_SUBM="ccc_msub ${TRUS_DIR}/batch/${TRUS_HPCC}.sh | grep -oe '\<[0-9]*\>'" 12 JOB_STAT=ccc_mpp 13 13 JOB_INFO=ccc_macct 14 JOB_KILL=ccc_mdel 15 16 [ -z "${TRUS_NPRO}" ] && TRUS_NPRO='16' 17 [ -z "${TIMEOUT}" ] && TIMEOUT='10800' 18 14 19 JOB_TIME="ccc_macct \${JOB_ID} | awk '/opa/ {print \$9}'" 15 JOB_DELE=ccc_mdel 20 JOB_PMEM="ccc_macct \${JOB_ID} | awk " 21 JOB_VMEM="ccc_macct \${JOB_ID} | awk " 16 22 17 ## Compiling & computing settings18 [ -z "$NPROC" ] && NPROC=1619 [ -z "${TIME_LIMI}" ] && TIME_LIMI=10800 # 3h20 21 ## Architecture environment (be careful with compliance with XIOS)22 ARCH_ENV=${DIR_XIOS}/arch/arch-${NEMO_HPCC}.env23 24 ## $CDO command to compare 2 datasets25 23 CDOD='cdo diffn' -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/config/arch_template.cfg
r5761 r5788 1 ## Copy to ./${NEMO_HPCC}.cfg, fill in according to your machine 1 ## HPC configuration template 2 ##--------------------------------------------------- 2 3 3 ## Super-computer configuration 4 ## Copy to root (./config) and rename it by keeping '.cfg' extension, then fill in according to your (super)computer 5 ## Will be called by its name without the extension 4 6 5 7 6 ## Architecture environment 7 ARCH_ENV='' ## (be careful with compliance with XIOS) 8 ## ex: "${DIR_XIOS}/arch/arch-${NEMO_HPCC}.env" 8 ## Mandatory 9 ##--------------------------------------------------- 10 11 ## (Super)Computer environment 12 ARCH_ENV='' ## *.env, *.path or modulesfile to source (let void if $PATH has already been setted) 13 ## ex: "${DIR_XIOS}/arch/arch-${TRUS_HPCC}.env" 14 15 ## Job commands (use with `eval ${JOB_....}`), may need to use "\${JOB_ID}" variable as the job ID returned from submission: 16 JOB_SUBM='' ## Submit a job and get his ID in return (will became ${JOB_ID}) 17 ## ex: `llsubmit ${TRUS_DIR}/batch/${TRUS_HPCC}.sh | grep -oe '\<[0-9]*\>'`, use absolute path 18 JOB_STAT='' ## Test state of the submitted job (true -> pending/running or false -> completed/crashed) 19 ## ex: "llq | grep \${JOB_ID}" 20 JOB_KILL='' ## Kill the submitted job 21 ## ex: "llcancel \${JOB_ID}" 22 23 ## Compiling (number of tasks) and computation (timeout in s) 24 [ -z "${TRUS_NPRO}" ] && TRUS_NPRO='1' 25 [ -z "${TIMEOUT}" ] && TIMEOUT='10800' 26 27 ## $CDO command to compare 2 datasets in argument 28 CDOD='' ## Essential to check a difference between NetCDF restart files 9 29 10 30 11 ## Available softwares (pattern to search before release number): 12 CDO='' ## CDO (Climate Data Operators) (string appears in $LOADEDMODULES or $PATH) 13 ## ex: 'cdo'/'nco'/... 14 COMPILER='' ## Fortran compiler (use with `$COMPILER --version`) 15 ## ex: 'ifort'/'gfortran'/'pgfortran'/... 16 MPI='' ## MPI libraries ( " " "" "" "" "" " " ) 17 ## ex: 'opempi'/'mpich'/... 18 NETCDF='' ## NetCDF " " ( " " "" "" "" "" " " ) 19 ## ex: 'netcdf42'/'netcdf-parallel'/... 31 ## Interest when monitoring HPC center 32 ##--------------------------------------------------- 33 34 ## Software releases (string pattern to match just before the number in $LOADEDMODULES or $PATH): 35 STR_CDOD='' ## CDO (Climate Data Operators) 36 ## ex: 'cdo'||'nco'||... 37 STR_LMPI='' ## MPI libraries 38 ## ex: 'openmpi'||'mpich'||... 39 STR_NCDF='' ## NetCDF " " 40 ## ex: 'netcdf'||'netcdf-parallel'||... 41 42 ## Fortran compiler version (use with `$COMPILER --version`) 43 CMPF='' ## ex: 'ifort'||'gfortran'||'pgfortran'||... 20 44 21 45 22 ## Job commands (works with job ID): 23 JOB_SUBM='' ## Submit a job && get his ID in return 24 ## ex: "cmd ${NEMO_TRUS}/batch/${NEMO_HPCC}.sh | awk '{print \$NF}'" 25 JOB_LIST='' ## List all submitted jobs to determine if a job is completed or not 26 JOB_INFO='' ## Get job informations to fill in a log file (job ID as argument) 27 JOB_TIME='' ## Get real CPU time computation with `eval "${JOB_INFO} ${JOB_ID} | ${JOB_TIME}"` 28 ## ex: "awk '/opa/ END{print \$9}'" 29 JOB_DELE='' ## Kill submitted job (job ID as argument) 46 ## Optional (can be void if no alternative or need) 47 ##--------------------------------------------------- 30 48 31 32 ## Compiling & computation 33 [ -z "$NPROC" ] && NPROC=1 ## Processing units for compilation 34 [ -z "${TIME_LIMI}" ] && TIME_LIMI='' ## Utmost time in seconds to wait for completed job 49 ## Computational performances of finished job (use with `eval ${JOB_....}`) 50 JOB_INFO='' ## Get job informations to fill in a logfile during computation 51 ## ex: "llq -x \${JOB_ID}" 52 JOB_TIME='' ## Get CPU elapsed time 53 ## ex: `grep IdrisMemMPI NEMO_CI_\${JOB_ID} | cut -d, -f1 | awk '{print \$(NF-1),\$NF}'` 54 JOB_PMEM='' ## Get max physical memory 55 ## ex: `grep IdrisMemMPI NEMO_CI_\${JOB_ID} | cut -d, -f3 | awk '{print \$3,\$4}'` 56 JOB_VMEM='' ## Get " " virtual "" -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/config/martin.cfg
r5685 r5788 2 2 ##------------------- 3 3 4 5 ## Global Directories:6 4 DIR_WORK=$CCCWORKDIR 7 5 DIR_SCRA=$SCRATCHDIR/trusting 8 6 DIR_STOR=$CCCSTOREDIR/trusting_sav 9 7 8 if [ -z "${TRUS_WKCY}" ]; then branch='trunk'; else branch=${TRUS_WKCY}; fi 9 TRUS_WKCY=${DIR_WORK}/NEMO/$branch/NEMOGCM 10 [ -z "${TRUS_REFE}" ] && TRUS_REFE='ORCA2_LIM_PISCES' 11 TRUS_BHMK=${DIR_STOR}/${TRUS_REFE}/${TRUS_WKCY} 12 TEST_DIR=${DIR_SCRA}/${TRUS_REFE}/${TRUS_WKCY}/trusting_${DATE} 10 13 11 ## NEMO overall installation: 12 [ -z "${NEMO_BRAN}" ] && NEMO_BRAN='trunk' 13 NEMO_HOME=${DIR_WORK}/NEMO/${NEMO_BRAN}/NEMOGCM 14 DIR_XIOS=${DIR_WORK}/XIOS/xios-1.0 14 15 15 ### Reference/'Trusting' configuration 16 [ -z "${REFE_CONF}" ] && REFE_CONF='ORCA2_LIM_PISCES' 17 TEST_CONF=${REFE_CONF}'_trust' 16 TRUS_TEST=${TRUS_REFE}'_trust' 17 [ ${TRUS_REFE} == 'ORCA2_LIM_PISCES' ] && TRUS_TEST='O2LP_trust' 18 [ ${TRUS_REFE} == 'ORCA1_LIM3_PISCES' ] && TRUS_TEST='O1L3P_trust' 19 18 20 KEYS_ADD=''; KEYS_DEL='' 19 [ ${REFE_CONF} == 'ORCA2_LIM_PISCES' ] && TEST_CONF='O2LP_trust' 20 [ ${REFE_CONF} == 'ORCA1_LIM3_PISCES' ] && TEST_CONF='O1L3P_trust' 21 if [ ${REFE_CONF} == 'ORCA2_LIM' ]; then 22 TEST_CONF='O2LA_trust' 21 22 if [ ${TRUS_REFE} == 'ORCA2_LIM' ]; then 23 TRUS_TEST='O2LA_trust' 23 24 KEYS_ADD='key_add key_agrif' && KEYS_DEL='key_del key_zdftmx' 24 25 fi 25 26 26 ### Inputs 27 NEMO_FORC=${DIR_WORK}/NEMO/FORC 28 [ -z "${NEMO_TARF}" ] && NEMO_TARF='ORCA2_LIM_nemo_v3.6.tar' 29 [ ${REFE_CONF} == 'ORCA1_LIM3_PISCES' ] && NEMO_TARF='INPUTS_ORCA1_LIM3_PISCES_V6.tar' 30 [ ${REFE_CONF} == 'AMM12' ] && NEMO_TARF='AMM12_v3.6.tar' 27 TRUS_FORC=${DIR_WORK}/NEMO/FORC 28 [ -z "${TRUS_TARF}" ] && TRUS_TARF='ORCA2_LIM_nemo_v3.6.tar' 29 [ ${TRUS_REFE} == 'ORCA1_LIM3_PISCES' ] && TRUS_TARF='INPUTS_ORCA1_LIM3_PISCES_V6.tar' 30 [ ${TRUS_REFE} == 'AMM12' ] && TRUS_TARF='AMM12_v3.6.tar' 31 31 32 33 ## Comparatives directories path: 34 REFE_DIR=${DIR_STOR}/${REFE_CONF}/${NEMO_BRAN} 35 TEST_DIR=${DIR_SCRA}/${REFE_CONF}/${NEMO_BRAN}/trusting_${DATE} 36 37 38 ## External softs directories: 39 DIR_XIOS=${DIR_WORK}/XIOS/xios-1.0 40 #DIR_OASI=${DIR_WORK}/OASIS 41 42 43 ## Mailing list to inform of a failure (-p|--publish option required) 44 [ -z "$EMAIL" ] && EMAIL='ntmlod@locean-ipsl.upmc.fr' 32 [ -z "$TRUS_MAIL" ] && TRUS_MAIL='ntmlod@locean-ipsl.upmc.fr' -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/config/romr005.cfg
r5696 r5788 2 2 ##------------------- 3 3 4 5 ## Global Directories:6 4 DIR_WORK=$WORKDIR 7 5 DIR_SCRA=$WORKDIR/NEMO/trusting 8 6 DIR_STOR=$WORKDIR/NEMO/trusting_sav 9 7 8 if [ -z "${TRUS_WKCY}" ]; then branch='trunk'; else branch=${TRUS_WKCY}; fi 9 TRUS_WKCY=${DIR_WORK}/NEMO/$branch/NEMOGCM 10 [ -z "${TRUS_REFE}" ] && TRUS_REFE='ORCA2_LIM_PISCES' 11 TRUS_BHMK=${DIR_STOR}/${TRUS_REFE}/$branch 12 TEST_DIR=${DIR_SCRA}/${TRUS_REFE}/$branch/trusting_${DATE} 10 13 11 ## NEMO overall installation: 12 [ -z "${NEMO_BRAN}" ] && NEMO_BRAN='trunk' 13 NEMO_HOME=${DIR_WORK}/NEMO/${NEMO_BRAN}/NEMOGCM 14 DIR_XIOS=${DIR_WORK}/XIOS/xios-1.0 14 15 15 ### Reference/'Trusting' configuration 16 [ -z "${REFE_CONF}" ] && REFE_CONF='ORCA2_LIM_PISCES' 17 TEST_CONF=${REFE_CONF}'_trust' 16 TRUS_TEST=${TRUS_REFE}'_trust' 17 [ ${TRUS_REFE} == 'ORCA2_LIM_PISCES' ] && TRUS_TEST='O2LP_trust' 18 [ ${TRUS_REFE} == 'ORCA1_LIM3_PISCES' ] && TRUS_TEST='O1L3P_trust' 19 18 20 KEYS_ADD=''; KEYS_DEL='' 19 [ ${REFE_CONF} == 'ORCA2_LIM_PISCES' ] && TEST_CONF='O2LP_trust' 20 [ ${REFE_CONF} == 'ORCA1_LIM3_PISCES' ] && TEST_CONF='O1L3P_trust' 21 if [ ${REFE_CONF} == 'ORCA2_LIM' ]; then 22 TEST_CONF='O2LA_trust' 21 22 if [ ${TRUS_REFE} == 'ORCA2_LIM' ]; then 23 TRUS_TEST='O2LA_trust' 23 24 KEYS_ADD='key_add key_agrif' && KEYS_DEL='key_del key_zdftmx' 24 25 fi 25 26 26 ### Inputs 27 NEMO_FORC=${DIR_WORK}/NEMO/FORC 28 [ -z "${NEMO_TARF}" ] && NEMO_TARF='ORCA2_LIM_nemo_v3.6.tar' 29 [ ${REFE_CONF} == 'ORCA1_LIM3_PISCES' ] && NEMO_TARF='INPUTS_ORCA1_LIM3_PISCES_V6.tar' 30 [ ${REFE_CONF} == 'AMM12' ] && NEMO_TARF='AMM12_v3.6.tar' 27 TRUS_FORC=${DIR_WORK}/NEMO/FORC 28 [ -z "${TRUS_TARF}" ] && TRUS_TARF='ORCA2_LIM_nemo_v3.6.tar' 29 [ ${TRUS_REFE} == 'ORCA1_LIM3_PISCES' ] && TRUS_TARF='INPUTS_ORCA1_LIM3_PISCES_V6.tar' 30 [ ${TRUS_REFE} == 'AMM12' ] && TRUS_TARF='AMM12_v3.6.tar' 31 31 32 33 ## Comparatives directories path: 34 REFE_DIR=${DIR_STOR}/${REFE_CONF}/${NEMO_BRAN} 35 TEST_DIR=${DIR_SCRA}/${REFE_CONF}/${NEMO_BRAN}/trusting_${DATE} 36 37 38 ## External softs directories: 39 DIR_XIOS=${DIR_WORK}/XIOS/xios-1.0 40 #DIR_OASI=${DIR_WORK}/OASIS 41 42 43 ## Mailing list to inform of a failure (-p|--publish option required) 44 [ -z "$EMAIL" ] && EMAIL='ntmlod@locean-ipsl.upmc.fr' 32 [ -z "$TRUS_MAIL" ] && TRUS_MAIL='ntmlod@locean-ipsl.upmc.fr' -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/config/user_template.cfg
r5761 r5788 1 ## Copy to ./${NEMO_USER}.cfg, fill in according to your installation 1 ## User configuration 2 ##--------------------------------------------------- 2 3 3 ## User configuration4 ## -------------------4 ## Copy to root (./config) and rename it by keeping '.cfg' extension, then fill in according to your installation 5 ## Will be called by its name without extension 5 6 6 7 7 ## Global Directories: 8 #DIR_WORK='' 9 #DIR_SCRA='' 10 #DIR_STOR='' 8 ## Mandatory 9 ##--------------------------------------------------- 10 11 ## Few settings can be changed from directly from command line to reduce the creation of user configuration file 12 ## See `./trusting.sh [-h|--help]` 13 14 ## Directories path 15 TRUS_BHMK='' ## Benchmark folder where target files for comparison have been stored 16 ## Inputs (all) : CPP_*, namelist_*, *.xml et inputs_file.txt (formed by first test) 17 ## Outputs (any) : *.stat, ocean.output, restarts 18 TEST_DIR='' ## Testing folder for computation 19 ## ex: ".../trusting_${DATE}" 20 TRUS_WKCY='' ## Working copy of branch test 21 ## ex: ".../NEMOGCM" 22 DIR_XIOS='' ## XIOS installation 23 24 ## NEMO configurations 25 TRUS_REFE='' ## Reference configuration to test 26 TRUS_TEST='' ## Testing configuration name 27 28 ## Forcing files 29 TRUS_FORC='' ## Directory with forcing archive to extract (combined use with ${TRUS_TARF}) or 30 ## " " "" all inputs files to copy into ${TEST_DIR} 11 31 12 32 13 ## NEMO overall installation: 14 [ -z "${NEMO_BRAN}" ] && NEMO_BRAN='' ## NEMO SVN branch to test 15 ## ex: 'trunk', 'nemo_v3_6_STABLE', ... 16 NEMO_HOME='' ## NEMOGCM directory (includes ./ARCH, ./CONFIG, ...) 17 ## ex: "${DIR_WORK}/NEMO/${NEMO_BRAN}/NEMOGCM", ... 18 ## ${NEMO_BRAN} must be consistent with URL branch that have been checkout by SVN to build ${NEMO_HOME} 33 ## Possible use 34 ##--------------------------------------------------- 35 36 ## Archive inputs name, let void if not requested 37 TRUS_TARF='' 38 39 ## CPP keys to add or remove from reference configuration 40 KEYS_ADD='' 41 KEYS_DEL='' 19 42 20 43 21 ### Reference/'Trusting' configuration 22 [ -z "${REFE_CONF}" ] && REFE_CONF='' ## Reference configuration to test 23 ## ex: 'ORCA2_LIM_PISCES', 'GYRE', ... 24 TEST_CONF='' ## Testing configuration name 25 KEYS_ADD=''; KEYS_DEL='' ## CPP keys to add or remove from reference configuration 26 ## ex: 'key_mpp_rep key_tide', ... 27 #if [ ${REFE_CONF} == '' ]; then 28 # TEST_CONF='' 29 # KEYS_ADD='key_add ...' && KEYS_DEL='key_del ...' 30 #elif ... 31 #else ... 32 #fi 44 ## Optional (can be void if no alternative or need) 45 ##--------------------------------------------------- 33 46 34 35 ### Inputs 36 NEMO_FORC='' ## Directory with forcing archive from DODS/ESGF server to extract or 37 ## " " "" all inputs files to copy into ${TEST_DIR} 38 [ -z "${NEMO_TARF}" ] && NEMO_TARF='' ## Archive inputs name, let blank if not requested 39 ## ex: 'ORCA2_LIM_nemo_v3.6.tar', 'AMM12_v3.6.tar', ... 40 #[ ${REFE_CONF} == '' ] && NEMO_TARF='' 41 42 43 ## Comparative directories path: 44 REFE_DIR='' ## "Standard" folder where benchmark result files have been stored for comparison 45 ## ex: "${DIR_STOR}/${REFE_CONF}/${NEMO_BRAN}" 46 TEST_DIR='' ## Testing folder for computation 47 ## ex: "${DIR_SCRA}/${REFE_CONF}/${NEMO_BRAN}/trusting_${DATE}" 48 49 50 ## External softs directories: 51 DIR_XIOS='' ## ex: '${DIR_WORK}/XIOS/xios-1.0' 52 #DIR_OASI='' ## ex: '${DIR_WORK}/OASIS' 53 54 55 ## Mailing list to inform of a failure (-p|--publish required) 56 [ -z "$EMAIL"] && EMAIL='' 47 ## Mailing list to notify of a failure (-p|--prod required) 48 TRUS_MAIL='' -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/trusting.env
r5695 r5788 2 2 3 3 4 ## Patterns ( time-stamp & filename)4 ## Patterns (UTC time-stamp & filename) 5 5 ##-------------------------------------- 6 6 dat=$( date ) 7 ## UTC time zone for timestamping8 7 export DATE=$( date -ud "$dat" +"%Y%m%d%Z%H%M" ) 9 export PATTERNAME=${NEMO_USER}'_'${NEMO_HPCC} 8 export PATTERNAME=${TRUS_USER}'_'${TRUS_HPCC} 9 10 ## Source (super)computer configuration 11 ##--------------------------------------------------- 12 . ${TRUS_DIR}/config/${TRUS_HPCC}.cfg 13 export ARCH_ENV 14 export JOB_INFO JOB_KILL JOB_PMEM JOB_STAT JOB_SUBM JOB_TIME JOB_VMEM 15 export TRUS_NPRO TIMEOUT 16 export CDOD 17 export STR_CDOD STR_LMPI STR_NCDF 18 export CMPF 10 19 11 20 ## Source user configuration 12 21 ##--------------------------------------------------- 13 . ${NEMO_TRUS}/config/${NEMO_USER}.cfg 14 export NEMO_HOME NEMO_BRAN 15 export REFE_CONF TEST_CONF KEYS_ADD KEYS_DEL 16 export NEMO_FORC NEMO_TARF 17 export REFE_DIR TEST_DIR 18 export DIR_XIOS #DIR_OASI 19 export EMAIL 20 21 ## Only interest for essential NEMO directories 22 NEMO_ARCH=${NEMO_HOME}/ARCH ; NEMO_CONF=${NEMO_HOME}/CONFIG 23 NEMO_ENGI=${NEMO_HOME}/NEMO 24 NEMO_EAGR=${NEMO_HOME}/EXTERNAL/AGRIF; NEMO_EIOI=${NEMO_HOME}/EXTERNAL/IOIPSL 25 NEMO_EFCM=${NEMO_HOME}/EXTERNAL/fcm 26 NEMO_TCMP=${NEMO_HOME}/TOOLS/COMPILE ; NEMO_TRBD=${NEMO_HOME}/TOOLS/REBUILD_NEMO 27 export NEMO_ARCH NEMO_CONF NEMO_ENGI 28 export NEMO_EAGR NEMO_EIOI NEMO_EFCM 29 export NEMO_TCMP NEMO_TRBD 30 31 ## Source super-computer configuration 32 ##--------------------------------------------------- 33 . ${NEMO_TRUS}/config/${NEMO_HPCC}.cfg 34 export CDO COMPILER MPI NETCDF 35 export JOB_SUBM JOB_LIST JOB_INFO JOB_TIME JOB_DELE 36 export NPROC TIME_LIMI 37 export ARCH_ENV 38 export CDOD 22 . ${TRUS_DIR}/config/${TRUS_USER}.cfg 23 export TRUS_BHMK TEST_DIR TRUS_REFE TRUS_TEST 24 export TRUS_WKCY DIR_XIOS 25 export TRUS_FORC TRUS_TARF 26 export KEYS_ADD KEYS_DEL 27 export TRUS_MAIL -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/trusting.sh
r5761 r5788 6 6 ## Set defaults 7 7 ##--------------------------------------------------- 8 NEMO_TRUS=$PWD 9 DEBUG=0; PUBLISH=0 10 IMOD=0; ST='FAILED' ## No modules system & 'FAILED' status 11 SVN_CMD='svn status' ## No update on directories 12 trus_help='trusting_help.txt' 8 DBG=0; PROD=0; HELP=0 9 TRUS_DIR=$PWD; TRUS_RSLT='FAILED'; SVN_CMD='svn status' # ('FAILED' result & no update on directories) 10 xios_mode='--full'; stdout_redir='>&' 11 revi=$( svn info | awk '(NR == 9) {print $NF}' ) 13 12 14 13 … … 16 15 ##--------------------------------------------------- 17 16 while [ $# -ne 0 ]; do 18 19 17 case $1 in 20 '-a'|'--archive') NEMO_TARF=$2 ; shift 2;; '-b'|'--branch ') NEMO_BRAN=$2; shift 2;;21 '-d'|'--debug ') set -vx; DEBUG=1; shift ;; '-e'|'--email ') EMAIL=$2; shift 2;;22 '-j'|'--job ') NPROC=$2 ; shift 2;; '-h'|'--help ') cat ${help_file}; exit 1;;23 '-m'|'--machine') NEMO_HPCC=$2 ; shift 2;; '-n'|'--newconf') TEST_CONF=$2; shift 2;;24 '-r'|'--refconf') REFE_CONF=$2 ; shift 2;; '-t'|'--time ') TIME_LIMI=$2; shift 2;;25 '-p'|'--p ublish') PUBLISH=1 ; shift ;; '-u'|'--user ') NEMO_USER=$2; shift 2;;26 '-v'|'--version') NEMO_VERS=$2 ; shift 2;; "*" ) cat ${help_file}; exit 1;;18 '-a'|'--archive') TRUS_TARF=$2 ; shift 2;; '-b'|'--branch' ) TRUS_WKCY=$2; shift 2;; 19 '-d'|'--debug' ) set -vx; DBG=1; shift ;; '-e'|'--email' ) TRUS_MAIL=$2; shift 2;; 20 '-j'|'--job' ) TRUS_NPRO=$2 ; shift 2;; '-h'|'--help' ) HELP=1 ; shift ;; 21 '-m'|'--machine') TRUS_HPCC=$2 ; shift 2;; '-n'|'--newconf') TRUS_TEST=$2; shift 2;; 22 '-r'|'--refconf') TRUS_REFE=$2 ; shift 2;; '-t'|'--time' ) TIMEOUT=$2 ; shift 2;; 23 '-p'|'--prod' ) PROD=1 ; shift ;; '-u'|'--user' ) TRUS_USER=$2; shift 2;; 24 '-v'|'--version') TRUS_VERS=$2 ; shift 2;; "*" ) HELP=1 ; shift ;; 27 25 esac 28 29 26 done 30 27 31 ## DEBUG options to speed up & expand verbosity of compilation32 xios_full='--full'; stderr_redir='>&'33 [ $DEBUG -eq 1 ] && { xios_full=''; stderr_redir='>'; }34 28 35 ## If -v|--version option has been set, modify default SVN action on directories 36 if [ $( echo ${NEMO_VERS} | grep "HEAD\|up\|update" ) ]; then 37 SVN_CMD='svn update -r HEAD' 38 elif [ $( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) ]; then 39 SVN_CMD='svn update -r '$( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) 40 elif [ $( echo ${NEMO_VERS} | grep -o '[0-9]*' ) ]; then 41 SVN_CMD='svn update -r '$( echo ${NEMO_VERS} | grep -o '[0-9]*' ) 42 fi 29 ## Initialization (HPC & user environment) 30 ##--------------------------------------------------- 31 if [[ ! $( find config -name ${TRUS_USER}'.cfg' && find config -name ${TRUS_HPCC}'.cfg' ) || $HELP -eq 1 ]]; then 32 cat trusting_help.txt 43 33 44 45 ## Initialization (super-computer & user environment) 46 ##--------------------------------------------------- 47 if [[ ! -z "${NEMO_HPCC}" && ! -z "${NEMO_USER}" ]]; then 48 49 if [ ! $( find ${NEMO_TRUS}/config -name ${NEMO_USER}'.cfg' ) ]; then 50 echo 'No configuration file (.cfg) in '${NEMO_TRUS}'/config for '${NEMO_USER} 51 exit 1 52 elif [ ! $( find ${NEMO_TRUS}/config -name ${NEMO_HPCC}'.cfg' ) ]; then 53 echo 'No configuration file (.cfg) in '${NEMO_TRUS}'/config for '${NEMO_HPCC} 54 exit 1 55 else 56 . ${NEMO_TRUS}/trusting.env 34 if [ $HELP -eq 0 ]; then 35 printf "\n\nAt least one configuration file is missing or misspelled: %s.cfg %s.cfg" \ 36 ${TRUS_USER} ${TRUS_HPCC} 57 37 fi 58 38 39 printf "\n\nContent of config folder:" 40 find config -name *.cfg | cut -d/ -f2 | xargs -n 3 printf "%-30s\t%-30s\t%-30s\n" 41 exit 1 59 42 else 60 cat ${NEMO_TRUS}/${trus_help} 61 exit 1 43 . trusting.env && . trusting_func.sh 44 45 ## DEBUG options to speed up & expand verbosity of compilation 46 [ $DBG -eq 1 ] && { xios_mode=''; stdout_redir='>'; } 47 48 ## If -v|--version option has been set, modify default SVN action on directories 49 if [ $( echo ${TRUS_VERS} | grep "HEAD\|up\|update" ) ]; then 50 SVN_CMD='svn update -r HEAD' 51 elif [ $( echo ${TRUS_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) ]; then 52 SVN_CMD='svn update -r '$( echo ${TRUS_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) 53 elif [ $( echo ${TRUS_VERS} | grep -o '[0-9]*' ) ]; then 54 SVN_CMD='svn update -r '$( echo ${TRUS_VERS} | grep -o '[0-9]*' ) 55 fi 56 62 57 fi 63 64 . ./trusting_func.sh65 58 66 59 67 60 ## Display contextual summary of trusting test 68 61 ##--------------------------------------------------- 69 echo 'NEMO Trusting test:' 70 echo '-------------------' 71 echo 'Testing '${TEST_CONF}' based on '${REFE_CONF}' from '${NEMO_BRAN} 72 echo 'NEMO installation: '${NEMO_USER} 73 echo 'Computing architecture: '${NEMO_HPCC} 74 echo 'Trusting SVN version: '$( svn info | awk '(NR == 9) {print $NF}' ) 62 echo 63 cat banner.txt 64 echo 65 echo '****************************************************************************************************' 66 echo '* *' 67 echo '* NEMO Trusting (Continuous Integration Tool) *' 68 echo "* ver.$revi *" 69 echo '* *' 70 echo '****************************************************************************************************' 71 echo 72 echo ' - Testing configuration '${TRUS_TEST}' based on '${TRUS_REFE} 73 echo ' - Working copy '${TRUS_WKCY} 74 echo ' - Benchmark folder '${TRUS_BHMK} 75 echo ' - (Super)Computer '${TRUS_HPCC} 76 echo ' - User installation '${TRUS_USER} 77 echo 75 78 76 79 … … 78 81 ##--------------------------------------------------- 79 82 print_step 'Timestamped testing directory' 80 mkdir -p ${TEST_DIR} ${ REFE_DIR}83 mkdir -p ${TEST_DIR} ${TRUS_BHMK} 81 84 cd ${TEST_DIR} && echo ${TEST_DIR} 82 85 init_files … … 92 95 ## Check softwares versions (after sourced arch environment) 93 96 ##--------------------------------------------------- 94 print_step 'Get softwares releases' 95 [ -e ${ARCH_ENV} ] && . ${ARCH_ENV} > /dev/null 96 [ $? -eq 0 ] && export IMOD=1 97 print_step 'Get testing environement' 97 98 get_soft_rel 98 99 cat model.log 100 env | sort > env.log 99 101 100 102 … … 103 105 print_step 'Compile XIOS' 104 106 cd ${DIR_XIOS} 105 eval ./make_xios ${xios_ full} --arch ${NEMO_HPCC} --job $NPROC\106 ${std err_redir} /dev/null107 eval ./make_xios ${xios_mode} --arch ${TRUS_HPCC} --job ${TRUS_NPRO} \ 108 ${stdout_redir} /dev/null 107 109 [ ! -e lib/libxios.a ] && get_out 1 || echo 'Success' 108 110 … … 110 112 ## NEMO compilation from scratch 111 113 ##--------------------------------------------------- 112 print_step "Compile ${ REFE_CONF} configuration"113 cd ${ NEMO_CONF}114 [[ -d ${T EST_CONF} && $DEBUG -eq 0 ]] && ./makenemo -n ${TEST_CONF} clean_config \114 print_step "Compile ${TRUS_REFE} configuration" 115 cd ${TRUS_WKCY}/CONFIG 116 [[ -d ${TRUS_TEST} && $DBG -eq 0 ]] && ./makenemo -n ${TRUS_TEST} clean_config \ 115 117 > /dev/null <<EOF 116 118 y 117 119 EOF 118 120 119 eval ./makenemo -n ${T EST_CONF} -r ${REFE_CONF} -m ${NEMO_HPCC} -j $NPROC\120 ${KEYS_ADD} ${KEYS_DEL} \121 ${std err_redir} /dev/null122 [ ! -e ${T EST_CONF}/BLD/bin/nemo.exe ] && get_out 2 || echo 'Success'121 eval ./makenemo -n ${TRUS_TEST} -r ${TRUS_REFE} -m ${TRUS_HPCC} -j ${TRUS_NPRO} \ 122 ${KEYS_ADD} ${KEYS_DEL} \ 123 ${stdout_redir} /dev/null 124 [ ! -e ${TRUS_TEST}/BLD/bin/nemo.exe ] && get_out 2 || echo 'Success' 123 125 124 126 … … 127 129 print_step 'Set job (copying or extracting inputs)' 128 130 cd ${TEST_DIR} 129 cp ${NEMO_CONF}/${TEST_CONF}/cpp_* .130 find ${ NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(_cfg\|.in\|opa\|_ref\|.xml\)' \131 -exec cp {} . \;131 cp ${TRUS_WKCY}/CONFIG/${TRUS_TEST}/cpp_* . 132 find ${TRUS_WKCY}/CONFIG/${TRUS_TEST}/EXP00 -regex '.*\(_cfg\|.in\|opa\|_ref\|.xml\)' \ 133 -exec cp {} . \; 132 134 get_inputs 133 135 [ $? -ne 0 ] && get_out 3 || echo 'Success' … … 137 139 ## Check inputs 138 140 ##--------------------------------------------------- 139 print_step 'Compare inputs to benchmark files'141 print_step 'Compare inputs' 140 142 diff_inputs 141 143 … … 151 153 152 154 153 ## Check job state & get comput ing timeif succeeded155 ## Check job state & get computation performances if succeeded 154 156 ##--------------------------------------------------- 155 157 print_step 'Test job state' 156 158 [[ ! -e time.step || $( grep 'E R R O R' ocean.output ) ]] && get_out 5 || echo 'Success' 157 print_step 'Get real CPU time' 158 get_cpu_time 159 print_step 'Get job performances' 160 get_time 161 get_memy 159 162 160 163 161 164 ## Check outputs 162 165 ##--------------------------------------------------- 163 ST='OK' ## 'OK' by default164 print_step 'Compare new outputs to benchmark files'166 TRUS_RSLT='OK' ## 'OK' by default 167 print_step 'Compare outputs' 165 168 diff_results 166 print_step 'Compare new restarts to benchmark files'169 print_step 'Compare restarts' 167 170 diff_restart 168 [ $ ST == 'FAILED' ] && get_out 8171 [ $TRUS_RSLT == 'FAILED' ] && get_out 8 169 172 170 173 -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/trusting_func.sh
r5701 r5788 3 3 4 4 ## Messenger filenames 5 FILE_DATE=mesg_01_date_$PATTERNAME.txt ; FILE_STAT=mesg_02_status_$PATTERNAME.txt 6 FILE_RESU=mesg_03_result_$PATTERNAME.txt ; FILE_NEMO=mesg_04_nemogcm_$PATTERNAME.txt 7 FILE_XIOS=mesg_05_xios_$PATTERNAME.txt ; FILE_COMP=mesg_06_compiler_$PATTERNAME.txt 8 FILE_MPIN=mesg_07_mpi_$PATTERNAME.txt ; FILE_NCDF=mesg_08_netcdf_$PATTERNAME.txt 9 FILE_INPT=mesg_09_inputs_$PATTERNAME.txt ; FILE_CPUT=mesg_10_cputime_$PATTERNAME.txt 10 FILE_COMM=mesg_11_comments_$PATTERNAME.txt; 11 12 ## Timestamped outcome & archive filenames 13 FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tar.gz 14 15 16 print_step() { printf "\nStep.....\n$1\n"; } 17 5 FILE_DATE=mesg_01_date_$PATTERNAME.txt ; FILE_TRUS_RSLT=mesg_02_result_$PATTERNAME.txt 6 FILE_STAT=mesg_03_state_$PATTERNAME.txt ; FILE_NEMO=mesg_04_nemo_$PATTERNAME.txt 7 FILE_XIOS=mesg_05_xios_$PATTERNAME.txt ; FILE_CMPF=mesg_06_compiler_$PATTERNAME.txt 8 FILE_LMPI=mesg_07_mpi_$PATTERNAME.txt ; FILE_NCDF=mesg_08_netcdf_$PATTERNAME.txt 9 FILE_INPT=mesg_09_inputs_$PATTERNAME.txt; FILE_TIME=mesg_10_time_$PATTERNAME.txt 10 FILE_MEMY=mesg_11_memory_$PATTERNAME.txt; FILE_NOTE=mesg_12_comments_$PATTERNAME.txt 11 12 ## Timestamped logfile & archive filenames 13 FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tgz 14 15 16 ## Functions in order of use 17 print_step() { 18 local char_nb=$( echo "$1" | wc -c ) 19 local outline=$( printf "%${char_nb}s" ) 20 21 printf "\nStep.....\n%s\n%s\n" "$1" ${outline// /-} 22 } 18 23 19 24 init_files() { 20 echo 'Date' > ${FILE_DATE}; echo 'Status' > ${FILE_STAT}21 echo ' Result' > ${FILE_RESU}; echo 'NEMOGCM rev.' >${FILE_NEMO}22 echo 'XIOS rev.' > ${FILE_XIOS}; echo 'Fortran compiler' > ${FILE_COMP}23 echo 'MPI libs' > ${FILE_MPIN}; echo 'NetCDF libs' >${FILE_NCDF}24 echo 'Input files' > ${FILE_INPT}; echo 'Real CPU time' > ${FILE_CPUT}25 echo ' Comments' > ${FILE_COMM}25 echo 'Date' > ${FILE_DATE}; echo 'Result' > ${FILE_TRUS_RSLT} 26 echo 'State' > ${FILE_STAT}; echo 'NEMOGCM rev.' > ${FILE_NEMO} 27 echo 'XIOS rev.' > ${FILE_XIOS}; echo 'Fortran compiler' > ${FILE_CMPF} 28 echo 'MPI libs' > ${FILE_LMPI}; echo 'NetCDF libs' > ${FILE_NCDF} 29 echo 'Input files' > ${FILE_INPT}; echo 'Elapsed time' > ${FILE_TIME} 30 echo 'Memory usage (P/V)' > ${FILE_MEMY}; echo 'Comments' > ${FILE_NOTE} 26 31 27 32 ## 'Failed' status with 'Unknown error' by default 28 echo $ST \ 33 echo $TRUS_RSLT \ 34 >> ${FILE_TRUS_RSLT} 35 echo 'Unknown error' \ 29 36 >> ${FILE_STAT} 30 echo 'Unknown error' \ 31 >> ${FILE_RESU} 32 } 33 37 } 34 38 35 39 get_date() { … … 37 41 local dat=$( date -ud "${DATE}" +"%F %R %Z" ) 38 42 39 echo $dat \43 echo $dat \ 40 44 >> ${FILE_DATE} 41 45 } 42 43 46 44 47 get_nemo_rev() { 45 48 local dir rev_loc 46 local rev=0 47 48 ## Loop on essential directories 49 for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \ 50 ${NEMO_EAGR} ${NEMO_EIOI} ${NEMO_EFCM} \ 51 ${NEMO_TCMP} ${NEMO_TRBD} \ 52 ${DIR_XIOS} ; do 49 local rev=0 list=( 'ARCH CONFIG NEMO EXTERNAL/AGRIF EXTERNAL/IOIPSL EXTERNAL/fcm TOOLS/COMPILE TOOLS/REBUILD_NEMO' ) 50 51 ## Loop on essential NEMO directories 52 for dir in $list ${DIR_XIOS}; do 53 53 54 54 ## For time being, just get revision from XIOS with no action on directory … … 62 62 fi 63 63 64 echo $dir && ${SVN_CMD} $ dir65 rev_loc=$( svn info $ dir | awk '(NR == 9) {print $NF}' )64 echo $dir && ${SVN_CMD} ${TRUS_WKCY}/$dir 65 rev_loc=$( svn info ${TRUS_WKCY}/$dir | awk '(NR == 9) {print $NF}' ) 66 66 67 67 ## Keep last rev. nb … … 75 75 } 76 76 77 78 77 get_soft_rel() { 79 78 local soft soft_rel 80 79 81 for soft in $CDO $COMPILER $MPI $NETCDF; do 80 ## Sourcing environment modulefile only if module function is set 81 [[ -e ${ARCH_ENV} && $( declare -F | grep ' module' ) ]] && . ${ARCH_ENV} 82 83 for soft in $CMPF ${STR_CDOD} ${STR_LMPI} ${STR_NCDF}; do 82 84 soft_rel='' 83 85 84 ## Software strings to identify depends on use (sed is case-insensitive) 85 if [ $IMOD -eq 1 ]; then 86 ## Modules system: next word after "$soft/" 87 soft_rel=$( echo $LOADEDMODULES | sed "s#.*$soft/\([0-9.a-z_]*\).*#\1#i" ) 88 ## LOADEDMODULES=netcdf/mpi/4.1.3:cdo/1.5.9:intel/2013.0 89 ## LOADEDMODULES=bullxmpi/1.2.8.2:netcdf/4.3.3.1_hdf5_parallel:cdo/1.6.7 90 else 91 ## Hard-coded paths: next word after "$soft" 92 soft_rel=$( echo $PATH | sed "s#.*$soft\([0-9.a-z_]*\).*#\1#i" ) 93 ## PATH=/smplocal/intel/impi/4.1.0.024/intel64/bin:/smplocal/pub/cdo/1.5.9/bin:/smplocal/pub/NetCDF/4.1.3/mpi/bin 94 ## PATH=/usr/local/netcdf-4.3.3.1_hdf5_parallel/bin:/opt/mpi/bullxmpi/1.2.8.2/bin:/usr/local/netcdf-utils-4.3.3.1_hdf5/bin 95 fi 86 ## Software release: next word after "$soft" in $PATH (case-insensitive) 87 soft_rel=$( echo $PATH | sed "s#.*$soft\([0-9.a-z_]*\).*#\1#i" ) 96 88 97 89 ## option --version would work for main compilers (gfortran, intel, pgfortran, ...) … … 106 98 107 99 sed -n 4p model.log \ 108 >> ${FILE_C OMP}100 >> ${FILE_CMPF} 109 101 sed -n 5p model.log \ 110 >> ${FILE_ MPIN}102 >> ${FILE_LMPI} 111 103 sed -n 6p model.log \ 112 104 >> ${FILE_NCDF} 113 105 } 114 106 115 116 107 get_inputs() { 117 108 # List archive content & extract it by default 118 local cmd_iol="tar -tvf ${ NEMO_FORC}/${NEMO_TARF}" cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}"109 local cmd_iol="tar -tvf ${TRUS_FORC}/${TRUS_TARF}" cmd_iof="tar -vxf ${TRUS_FORC}/${TRUS_TARF}" 119 110 120 111 ## List & copy files in case of personal inputs 121 [ -z "${ NEMO_TARF}" ] && { cmd_iol="ls ${NEMO_FORC}/*"; cmd_iof="\cp ${NEMO_FORC}/* ."; }112 [ -z "${TRUS_TARF}" ] && { cmd_iol="ls ${TRUS_FORC}/*"; cmd_iof="\cp ${TRUS_FORC}/* ."; } 122 113 123 114 ${cmd_iol} > inputs_list.txt 124 115 ${cmd_iof} > /dev/null 125 116 } 126 127 117 128 118 diff_inputs() { … … 135 125 136 126 ## Continue even if input file is not in here (see after) 137 if [ -e ${ REFE_DIR}/$file ]; then dif=$( diff -q $file ${REFE_DIR}/$file ); else dif=0; fi127 if [ -e ${TRUS_BHMK}/$file ]; then dif=$( diff -q $file ${TRUS_BHMK}/$file ); else dif=0; fi 138 128 139 129 ## Pass over useless file omission in benckmark directory 140 if [[ -n "$dif" && "$dif" != '0' ]]; then 141 mesg='Different'; echo $dif; files_list+=$file' ' 142 fi 143 130 [[ -n "$dif" && "$dif" != '0' ]] && ( mesg='Different'; echo $dif; files_list+=$file' ' ) 144 131 done 145 132 146 133 [ $mesg == 'Same' ] && echo $mesg 147 echo $mesg \134 echo $mesg \ 148 135 >> ${FILE_INPT} 149 136 150 137 ## List different files for web comment 151 138 [ -n "${files_list}" ] && echo 'Inputs : '${files_list}'differ<br>' \ 152 >> temp_${FILE_COMM} 153 } 154 139 >> temp_${FILE_NOTE} 140 } 155 141 156 142 job_pending() { … … 160 146 161 147 ## Append a log file while pending 162 while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI}]]; do148 while [[ $( eval ${JOB_STAT} ) && ${time_elapsed} -lt $TIMEOUT ]]; do 163 149 printf "\n%s\n" ${outline// /#} \ 164 150 >> computation.log 165 ${JOB_INFO} ${JOB_ID}\151 eval ${JOB_INFO} \ 166 152 >> computation.log 167 153 sleep ${time_increment} … … 172 158 173 159 ## Kill remaining job & stop the test if it's too long 174 [ ${time_elapsed} -eq ${TIME_LIMI} ] && { ${JOB_DELE} ${JOB_ID} &> /dev/null; get_out 6; } 175 } 176 160 [ ${time_elapsed} -eq $TIMEOUT ] && { eval ${JOB_DELE} &> /dev/null; get_out 6; } 161 } 177 162 178 163 diff_results() { … … 182 167 ## Simple diff 183 168 for file in 'ocean.output' *.stat; do 184 185 169 ## Stop if no benchmark files (ocean.output, eventual stat files) 186 [ ! -e ${ REFE_DIR}/$file ] && { export ST='FAILED'; get_out 7; }187 188 diff -q $file ${ REFE_DIR}/$file170 [ ! -e ${TRUS_BHMK}/$file ] && { export TRUS_RSLT='FAILED'; get_out 7; } 171 172 diff -q $file ${TRUS_BHMK}/$file 189 173 190 174 ## Continue even if it differs 191 [ $? -ne 0 ] && { export ST='FAILED'; mesg='Different'; files_list+=$file' '; }175 [ $? -ne 0 ] && { export TRUS_RSLT='FAILED'; mesg='Different'; files_list+=$file' '; } 192 176 done 193 177 … … 196 180 ## List different files for web comment 197 181 [ -n "${files_list}" ] && echo 'Results : '${files_list}'differ<br>' \ 198 >> temp_${FILE_COMM} 199 } 200 182 >> temp_${FILE_NOTE} 183 } 201 184 202 185 diff_restart() { … … 205 188 206 189 ## Stop if no benchmark files (ie time.step) 207 [ ! -e ${ REFE_DIR}/time.step ] && { export ST='FAILED'; get_out 7; }208 time_step=$( cat ${ REFE_DIR}/time.step | tr -d [:space:] )190 [ ! -e ${TRUS_BHMK}/time.step ] && { export TRUS_RSLT='FAILED'; get_out 7; } 191 time_step=$( cat ${TRUS_BHMK}/time.step | tr -d [:space:] ) 209 192 210 193 ## Find all restart files to rebuild 211 194 if [ $( find -regex ".*_restart.*[0-9]\.nc" -print -quit ) ]; then 212 base_name=$( find -regex ".*_restart.*[0-9]\.nc" \213 | sed "s#^\./\(.*\)_[0-9]*_restart.*#\1#" | sort -u )214 list_comp=$( find -regex ".*_restart.*[0-9]\.nc" \215 | sed "s#^.*\(restart[a-z_]*\)_[0-9].*\.nc#\1#" | sort -u )216 list_tmsp=$( find -regex ".*_restart.*[0-9]\.nc" \217 | sed "s#^.*\([0-9]\{8\}\)_restart.*#\1#" | sort -u )195 base_name=$( find -regex ".*_restart.*[0-9]\.nc" \ 196 | sed "s#^\./\(.*\)_[0-9]*_restart.*#\1#" | sort -u ) 197 list_comp=$( find -regex ".*_restart.*[0-9]\.nc" \ 198 | sed "s#^.*\(restart[a-z_]*\)_[0-9].*\.nc#\1#" | sort -u ) 199 list_tmsp=$( find -regex ".*_restart.*[0-9]\.nc" \ 200 | sed "s#^.*\([0-9]\{8\}\)_restart.*#\1#" | sort -u ) 218 201 219 202 ## Loop on each time step 220 203 for tmsp in ${list_tmsp}; do 204 221 205 for comp in ${list_comp}; do 222 206 file=${base_name}_${tmsp}_${comp} … … 224 208 225 209 if [ ${nb_dom} -gt 1 ]; then 226 ${ NEMO_TRBD}/rebuild_nemo -t $NPROC$file ${nb_dom} > /dev/null227 [ $? -eq 0 ] && rm -f ${file}_[0-9]*.nc > /dev/null210 ${TRUS_WKCY}/TOOLS/REBUILD_NEMO/rebuild_nemo -t ${TRUS_NPRO} $file ${nb_dom} > /dev/null 211 [ $? -eq 0 ] && rm -f ${file}_[0-9]*.nc > /dev/null 228 212 elif [ ${nb_dom} -eq 0 ]; then 229 export ST='FAILED' && get_out 8213 export TRUS_RSLT='FAILED' && get_out 8 230 214 fi 231 215 … … 234 218 235 219 ## Stop if no benchmark files (restart file) 236 if [ -e ${ REFE_DIR}/$file.nc ]; then220 if [ -e ${TRUS_BHMK}/$file.nc ]; then 237 221 238 222 ## UNIX `cmp` not suitable (timestamp in .nc file) 239 dif=$( $CDOD $file.nc ${ REFE_DIR}/$file.nc 2> /dev/null \223 dif=$( $CDOD $file.nc ${TRUS_BHMK}/$file.nc 2> /dev/null \ 240 224 | awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n' ) 241 225 242 226 ## CDO can return void stdout with no difference 243 227 if [[ -n "$dif" && $( echo $dif | awk '{print $1}' ) -ne 0 ]]; then 244 export ST='FAILED'245 files_list+=$comp' ' && let dif t+=$( echo $dif | awk '{print $1}' )228 export TRUS_RSLT='FAILED' 229 files_list+=$comp' ' && let dif_sum+=$( echo $dif | awk '{print $1}' ) 246 230 echo $file.nc': '$dif 247 231 fi 248 232 249 233 else 250 export ST='FAILED' && get_out 7234 export TRUS_RSLT='FAILED' && get_out 7 251 235 fi 252 236 … … 256 240 257 241 done 242 258 243 done 259 244 260 245 ## List different files for web comment with sum of different parameters 261 if [ $ dift-ne 0 ]; then262 echo 'Restarts: '${files_list}$ dift' record(s) differ<br>' \263 >> temp_${FILE_ COMM}246 if [ ${dif_sum} -ne 0 ]; then 247 echo 'Restarts: '${files_list}${dif_sum}' record(s) differ<br>' \ 248 >> temp_${FILE_NOTE} 264 249 else 265 250 echo 'Same' … … 267 252 268 253 else 269 export ST='FAILED' 270 fi 271 272 } 273 274 275 get_cpu_time() { 276 ## Interest for checking unusual time computing 277 local real_cpu_time=$( eval ${JOB_TIME} ) 278 279 echo ${real_cpu_time} | tee -a ${FILE_CPUT} 280 } 281 254 export TRUS_RSLT='FAILED' 255 fi 256 257 } 258 259 get_time() { 260 ## Interest for checking unusual time computation 261 local time_cpu=$( eval ${JOB_TIME} ) 262 263 printf "Elapsed time: " 264 echo ${time_cpu} | tee -a ${FILE_TIME} 265 } 266 267 get_memy() { 268 ## Interest for checking unusual memory usage 269 local memory_pmax=$( eval ${JOB_PMEM} ) memory_vmax=$( eval ${JOB_VMEM} ) 270 271 printf "Memory max usage (physical/virtual): " 272 echo ${memory_pmax}' / '${memory_vmax} | tee -a ${FILE_MEMY} 273 } 282 274 283 275 comments() { … … 286 278 287 279 if [ -e ocean.output ]; then 288 ## 'W A R N I N G' by default280 ## 'W A R N I N G' pattern by default 289 281 opat="-A2 \"^ $state\"" 290 282 [ "$state" == 'E R R O R' ] && opat="-A4 \"$state\"" … … 292 284 ## Select first occurence for web comment 293 285 line=$( eval grep -m1 $opat ocean.output | tr -d '\n' ) 294 echo $line 295 fi 296 297 printf "$line<br>" \ 298 >> temp_${FILE_COMM} 299 } 300 301 302 mesg_make() { 286 fi 287 288 [ -n "$line" ] && ( echo $line; printf "$line<br>" \ 289 >> temp_${FILE_NOTE} ) 290 } 291 292 log_make() { 303 293 ## Format comments for web 304 [ -e temp_${FILE_ COMM} ] && cat temp_${FILE_COMM} | tr -d '\n' | sed 's/<br>$//' \305 >> ${FILE_ COMM}306 307 ## Construct txt file for webwith all messenger files294 [ -e temp_${FILE_NOTE} ] && cat temp_${FILE_NOTE} | tr -d '\n' | sed 's/<br>$//' \ 295 >> ${FILE_NOTE} 296 297 ## Construct txt file with all messenger files 308 298 paste -d ';' mesg_*.txt | tee ${FILE_TRUS} 309 299 } 310 300 311 312 mesg_publish() { 301 prod_publish() { 313 302 local cmd 314 315 ## Production mode (-p|--publish) 316 if [ $PUBLISH -eq 1 ]; then 317 318 ## Create or append historical trusting file 319 if [ -f ${REFE_DIR}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi 303 local rev=$( awk '/NEMOGCM/ {print $NF}' model.log ) 304 305 ## Production mode (-p|--prod) 306 if [ $PROD -eq 1 ]; then 307 308 ## Create or append trusting logfile 309 if [ -f ${TRUS_BHMK}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi 320 310 321 311 $cmd ${FILE_TRUS} \ 322 >> ${ REFE_DIR}/trusting_$PATTERNAME.txt312 >> ${TRUS_BHMK}/trusting_$PATTERNAME.txt 323 313 324 314 ## Send mail only when FAILED 325 if [[ ! -z "$ EMAIL" && $ST == 'FAILED' ]]; then315 if [[ ! -z "$TRUS_MAIL" && $TRUS_RSLT == 'FAILED' ]]; then 326 316 327 317 ## Content 328 cat <<END_MAIL \318 cat <<END_MAIL \ 329 319 > trusting.mail 330 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 331 332 Dear $USER, 333 334 The trusting sequence for ${REFE_CONF} has failed. 335 Directory: ${TEST_DIR} 336 Archive created: ${FILE_ARCH} in ${REFE_DIR} 337 338 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 339 340 `cat ${TEST_DIR}/${FILE_TRUS}` 320 Dear all, 321 322 323 The trusting sequence has not completed successfully on new configuration ${TRUS_TEST} based on ${TRUS_REFE}. 324 325 Here is the model summary: 326 `cat model.log` 327 328 First checking would be on the trusting environment files: 329 ${TRUS_USER}.cfg & ${TRUS_HPCC}.cfg 330 331 For more details, look into the testing directory at: 332 ${TEST_DIR} 333 334 An archive has been created to share the questionable configuration for further studies: 335 ${TRUS_BHMK}/${FILE_ARCH} 336 341 337 END_MAIL 342 338 343 339 ## Send with detailed subject 344 mail -s "[NEMO Trusting ${NEMO_HPCC} ${REFE_CONF} ${NEMO_BRAN}] $ST $ERR" $EMAIL \340 mail -s "[NEMO Trusting][$rev][${TRUS_WKCY}][${TRUS_REFE}] $TRUS_RSLT $ERR" $TRUS_MAIL \ 345 341 < trusting.mail 346 342 fi … … 349 345 } 350 346 351 352 347 get_out() { 353 348 local time_step=0 … … 355 350 ERR=$1 356 351 357 print _step 'End of trusting test'352 printf "\n\nEnd of test\n" 358 353 359 354 ## In case of compilation error 360 355 cd ${TEST_DIR} 361 356 362 if [ $ ST == 'FAILED' ]; then357 if [ $TRUS_RSLT == 'FAILED' ]; then 363 358 echo 'Failure' 364 359 … … 369 364 ## Submission 370 365 '3') ERR='Missing input files' ;; '4') ERR='Job submission error' ;; 371 ## Running366 ## Computation 372 367 '5') ERR='Crashed at time step' ;; '6') ERR='Exceeded time limit' ;; 373 368 ## Results 374 369 '7') ERR='Missing previous outputs';; '8') ERR='New outputs differ' ;; 375 ## Undefined376 "*") ERR='Unknown error' ;;377 370 esac 378 371 … … 384 377 if [ "$ERR" == 'Crashed at time step' ]; then 385 378 comments 'E R R O R' 386 [ -e time.step ] && time_step=$( cat time.step | tr -d ' ')379 [ -e time.step ] && time_step=$( grep -o [0-9]* time.step ) 387 380 ERR+=' '$time_step 388 381 else 389 382 comments 'W A R N I N G' 390 [ "$ERR" == 'Exceeded time limit' ] && ERR+=' '$(( ${TIME _LIMIT}/3600 ))'h'383 [ "$ERR" == 'Exceeded time limit' ] && ERR+=' '$(( ${TIMEOUTT}/3600 ))'h' 391 384 fi 392 385 393 386 ## Last messenger files 394 387 export ERR 395 sed -i "2 s/.*/$ST/" ${FILE_STAT}; sed -i "2 s/.*/$ERR/" ${FILE_RESU} 396 397 ## Save tested configuration if trusting failed in production mode (-p|--publish) 398 if [[ $ST == 'FAILED' && $PUBLISH -eq 1 ]]; then 399 echo 'Creating archive '${FILE_ARCH}' under '${REFE_DIR} 400 tar -czf ${REFE_DIR}/${FILE_ARCH} * ${NEMO_HOME}/CONFIG/${TEST_CONF}/MY_SRC \ 401 ${NEMO_HOME}/CONFIG/${TEST_CONF}/WORK 402 fi 403 404 ## Share trusting result (.txt file/mail) 405 print_step 'Trusting digest' 406 mesg_make 407 mesg_publish 388 sed -i "2 s/.*/$TRUS_RSLT/" ${FILE_TRUS_RSLT}; sed -i "2 s/.*/$ERR/" ${FILE_STAT} 389 390 ## Save tested configuration if trusting failed in production mode (-p|--prod) 391 if [[ $TRUS_RSLT == 'FAILED' && $PROD -eq 1 ]]; then 392 echo 'Creating archive '${FILE_ARCH}' under '${TRUS_BHMK} 393 tar -czf ${TRUS_BHMK}/${FILE_ARCH} * \ 394 -C ${TRUS_WKCY}/CONFIG/${TRUS_TEST}/MY_SRC . \ 395 -C ${TRUS_WKCY}/CONFIG/${TRUS_TEST} cpp_${TRUS_TEST}.fcm 396 fi 397 398 ## Logfile construct & eventual sending of notification email 399 printf "\nTrusting digest:\n----------------\n" 400 log_make 401 prod_publish 408 402 409 403 exit 0 -
branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/trusting_help.txt
r5761 r5788 5 5 Mandatory (default settings): 6 6 Files to source in ./config ($ARCH.cfg & $USER.cfg) 7 -m,--machine Architecture settings (environment configuration)8 -u,--user User "" (NEMO directories & settings)7 -m,--machine Architecture environment (compilation & computation settings) 8 -u,--user User "" (NEMO & XIOS installation) 9 9 10 Other options (modify default settings) 10 11 SVN working copy: 11 -b,--branch SVN branch to test (trunk|nemo_v3_6_STABLE|...12 -v,--version Action on working copy ('svnstatus' by default)13 '{'YYYY-MM-DD'}' Update to revision available at this date14 [0-9]* "" "" revision number15 HEAD|up|update "" "" last revision16 st|status Show differences with repository12 -b,--branch SVN branch name to test (trunk|nemo_v3_6_STABLE|...) 13 -v,--version Action on working copy ('status' by default) 14 '{'YYYY-MM-DD'}' Update to revision available at this date 15 [0-9]* "" "" revision number 16 HEAD|up|update "" "" last revision 17 st|status Show differences with repository 17 18 18 NEMO Configuration:19 -a,--archive Forcing input files archive to extract downloaded from ESGF/DODS server20 -n,--new conf Testing configuration created in ./CONFIG21 -r,--ref conf Reference " "19 NEMO configuration: 20 -a,--archive Forcing input files archive to extract downloaded from ESGF/DODS server 21 -n,--new_conf Testing configuration created in ./CONFIG 22 -r,--ref_conf Reference " " 22 23 23 24 Compiling & computation: 24 -j,--job Number of processes for compiling25 -t,--time Timeout for computation (s)25 -j,--job Number of processes for compiling 26 -t,--time Timeout for computation (s) 26 27 27 28 Output control: 28 -d,--debug Verbose output for debugging purpose (`set -vx`)29 -e,--email Email trusting digest report to following addresses only on failure30 -h,--help Print this help & exit31 -p,--p ublish Share trusting result29 -d,--debug Verbose output for debugging purpose (`set -vx`) 30 -e,--email Email trusting digest report to following addresses only on failure 31 -h,--help Print this help & exit 32 -p,--prod Production mode (append trusting logfile)
Note: See TracChangeset
for help on using the changeset viewer.