New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
all_functions.sh in utils/CI/sette_ticket2304 – NEMO

source: utils/CI/sette_ticket2304/all_functions.sh @ 12290

Last change on this file since 12290 was 12290, checked in by mathiot, 4 years ago

use rsync for sync instead of cp and remove sync of cpp key (ticket #2304)
  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 18.5 KB
Line 
1######################################################
2# Author : Simona Flavoni for NEMO
3# Contact : sflod@locean-ipsl.upmc.fr
4#
5# ----------------------------------------------------------------------
6# NEMO/SETTE , NEMO Consortium (2010)
7# Software governed by the CeCILL licence     (NEMOGCM/NEMO_CeCILL.txt)
8# ----------------------------------------------------------------------
9#
10# Some scripts called by sette.sh
11# all_functions.sh   : all functions used by sette.sh 
12######################################################
13#set -x
14#set -o posix
15#set -u
16#set -e
17#+
18#
19# ================
20# all_functions.sh
21# ================
22#
23# ----------------------------------------------
24# Set of functions used by sette.sh (NEMO tests)
25# ----------------------------------------------
26#
27# SYNOPSIS
28# ========
29#
30# ::
31#
32#  $ ./set_namelist INPUT_NAMELIST VARIABLE VALUE
33#  $ post_test_tidyup
34#
35#
36# DESCRIPTION
37# ===========
38#
39# function superegrep
40#   input variable value
41#
42# function set_namelist
43#   input namelist_name variable value
44#   output namelist
45#
46# function post_test_tidyup
47#   creates nemo_validation tree, and save output & debug files
48#   this function creates tree of validation in NEMO_VALIDATION_DIR as follows :
49#
50# NEMO_VALIDATION_DIR/WCONFIG_NAME/WCOMPILER_NAME/REVISION_NUMBER(or DATE)/TEST_NAME
51#
52# NEMO_VALIDATION_DIR           : is choosen in param.cfg
53#
54# WCONFIG_NAME                  : set by makenemo at the moment of compilation
55#
56# WCOMPILER_NAME                : set by makenemo at the moment of compilation
57#
58# REVISION_NUMBER(or DATE)      : revision number by svn info, if problems with svn date is taken
59#
60# TEST_NAME                     : set in sette.sh for each configuration to be tested (directory TEST_NAME is created under ${NEW_CONF} directory )
61#
62# EXAMPLES
63# ========
64#
65# ::
66#
67#  $ ./set_namelist namelist          nn_itend        75
68#  $ ./set_namelist namelist_ice      cn_icerst_in  \"00101231_restart_ice\"
69#  $ post_test_tidyup
70#
71#
72# TODO
73# ====
74#
75# option debug
76#
77#
78# EVOLUTIONS
79# ==========
80#
81# $Id$
82#
83#   * creation
84#-
85# function to find namelists parameters
86supergrep () {
87            grep "^ *$1 *=" $2 | sed -e "s% *\!.*%%"
88    }
89
90usage=" Usage : set_namelist input_namelist variable_name value"
91usage=" if value is a string ths is neede syntax : ./set_namelist namelist_name var_name \"new_value\" "
92
93# sync MYSRC files
94sync_mysrc() {
95lREF=cfgs/$1    # reference
96lCFG=cfgs/$2    # target
97if [ -d $lREF/MY_SRC ] ; then
98   rsync $lREF/MY_SRC/* $lCFG/MY_SRC/*
99fi
100}
101
102# sync namelist files and xios related files
103sync_expcfg() {
104set -x
105lREF=cfgs/$1
106lCFG=cfgs/$2
107if [ -d $lREF/EXPREF ] ; then
108   rsync $lREF/EXPREF/*namelist*_cfg  $lCFG/EXP00/.
109   rsync $lREF/EXPREF/iodef.xml       $lCFG/EXP00/.
110   rsync $lREF/EXPREF/context_*.xml   $lCFG/EXP00/.
111   rsync $lREF/EXPREF/file_def_*.xml  $lCFG/EXP00/.
112fi
113set +x
114}
115
116# define validation dir
117set_valid_dir () {
118    LANG=en_US
119    REVISION_NB=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${SETTE_DIR}/.. | grep "Last Changed Rev" | awk '{print $NF}'`
120    if [ ${#REVISION_NB} -eq 0 ]
121    then
122        echo "some problems with ${SVN_CMD} info command"
123        echo "some problems with ${SVN_CMD} info command" >> ${SETTE_DIR}/output.sette
124        REVISION_NB=`date +%Y%m%d`
125        echo "put in ${REVISION_NB} date"
126        echo "put in ${REVISION_NB} date" >> ${SETTE_DIR}/output.sette
127    else
128    echo "value of revision number of NEMOGCM: ${REVISION_NB}"
129    fi
130    [ `${SVN_CMD} status -q ${SETTE_DIR}/../{cfg,tests,src} | wc -l` -ge 1 ] && REVISION_NB=${REVISION_NB}+
131    NEMO_VALID=${NEMO_VALIDATION_DIR}/W${NEW_CONF}/${CMP_NAM}/${REVISION_NB}/${TEST_NAME}
132}
133
134# clean valid dir (move old ocean_output/run.stat and tracer to avoid checking them in case something wrong happen.
135clean_valid_dir () {
136#   set_valid_dir # already done in sette_ref/sette_test
137   echo "validation directory is : $NEMO_VALID"
138   if [ -d $NEMO_VALID ] ; then
139      [ -f ${NEMO_VALID}/ocean.output ] && mv ${NEMO_VALID}/ocean.output ${NEMO_VALID}/ocean.output_old
140      [ -f ${NEMO_VALID}/run.stat ]     && mv ${NEMO_VALID}/run.stat     ${NEMO_VALID}/run.stat_old
141      [ -f ${NEMO_VALID}/tracer.stat ]  && mv ${NEMO_VALID}/tracer.stat  ${NEMO_VALID}/tracer.stat_old
142   fi
143}
144
145# function to set namelists parameters
146set_namelist () {
147   minargcount=3
148   if [ ${#} -lt ${minargcount} ]
149   then
150      echo "not enough arguments for set_namelist"
151      echo "${usage}"
152      exit 1
153   fi
154   unset minargcount
155   if [  ! -f ${SETTE_DIR}/output.sette ] ; then
156                touch ${SETTE_DIR}/output.sette
157        fi
158
159        echo "executing script : set_namelist $@" >> ${SETTE_DIR}/output.sette
160        echo "################" >> ${SETTE_DIR}/output.sette
161     
162   VAR_NAME=$( supergrep $2 ${EXE_DIR}/$1 )
163   if [ ${#VAR_NAME} -eq 0 ] 
164   then
165      echo "doing \"set_namelist $@\". "  >> ${SETTE_DIR}/output.sette
166      echo "variable: \"$2\" not found in \"${EXE_DIR}/$1\" "  >> ${SETTE_DIR}/output.sette
167                NAMREF=$( basename $1 _cfg )_ref
168      echo "doing more : search in ${EXE_DIR}/$NAMREF " >> ${SETTE_DIR}/output.sette
169                VAR_NAME=$( supergrep $2 ${EXE_DIR}/$NAMREF )
170           if [ ${#VAR_NAME} -eq 0 ] 
171           then
172                    echo " variable $VAR_NAME not found in ${EXE_DIR}/$1 nor in ${EXE_DIR}/$NAMREF "
173                    echo " check your variable name "
174            echo "exit"
175            echo "error in executing script : set_namelist $@" >> ${SETTE_DIR}/output.sette
176            echo "....." >> ${SETTE_DIR}/output.sette
177            exit 1
178                fi
179                LINEVAR=$( grep -s -n "$VAR_NAME" ${EXE_DIR}/$NAMREF | awk -F: '{ { print $1} }' )
180                echo " $VAR_NAME found in ${EXE_DIR}/$NAMREF at line $LINEVAR " >> ${SETTE_DIR}/output.sette
181
182#   search for namelist group name
183                NAMGRP=$( head -n$LINEVAR ${EXE_DIR}/$NAMREF | grep --line-buffered "^&nam" | tail -1 | awk -F" " '{ { print $1} }' ) 
184                echo " variable $VAR_NAME will be added in $NAMGRP namelist-group of namelist file ${EXE_DIR}/$1 " >> ${SETTE_DIR}/output.sette
185
186# check if namelist group present in namelist_cfg
187# if missing group is added at the end of namelist_cfg
188                NGRP=$(grep ${NAMGRP} ${EXE_DIR}/$1 | wc -l )
189                if [ ${NGRP} -eq 0 ]; then
190                   echo ''
191                   echo "+++++ Group ${NAMGRP} containing ${2} is missing in ${EXE_DIR}/$1 +++++ "     
192                   echo "+++++ Group ${NAMGRP}                 is added   in ${EXE_DIR}/$1 +++++ "     
193                   echo ''
194                   echo "&${NAMGRP}" >> ${EXE_DIR}/$1
195                   echo "/"          >> ${EXE_DIR}/$1
196                fi
197
198# Add $VARNAME in namelist file ${EXE_DIR}/$1 in namelist group $NAMGRP
199# on mac osx, replace sed --posix by gsed (available with mac port)
200                sed --posix "/${NAMGRP} /a\ ${VAR_NAME} " ${EXE_DIR}/$1 > ${EXE_DIR}/$1.tmp || gsed --posix "/${NAMGRP} /a\ ${VAR_NAME} " ${EXE_DIR}/$1 > ${EXE_DIR}/$1.tmp
201
202# if file not empty replace ${EXE_DIR}/$1
203               if [ -s ${EXE_DIR}/$1.tmp ] ; then
204                   mv ${EXE_DIR}/$1.tmp ${EXE_DIR}/$1 
205                else
206                echo "file ${EXE_DIR}/$1.tmp is empty. sed command went wrong "; exit 200
207                fi
208   fi
209
210        ARGS_LST="${@:3}"
211        sed -e "s;${VAR_NAME}.*;${VAR_NAME};" ${EXE_DIR}/$1 > ${EXE_DIR}/$1.tmp
212        mv ${EXE_DIR}/$1.tmp ${EXE_DIR}/$1
213        sed -e "s;${VAR_NAME};$2=${ARGS_LST};"  ${EXE_DIR}/$1 > ${EXE_DIR}/$1.tmp
214        mv ${EXE_DIR}/$1.tmp ${EXE_DIR}/$1
215
216        echo "finished script : set_namelist $@" >> ${SETTE_DIR}/output.sette
217        echo "++++++++++++++++" >> ${SETTE_DIR}/output.sette
218        echo "                " >> ${SETTE_DIR}/output.sette
219}
220
221
222# function to tidy up after each test and populate the NEMO_VALIDATION store
223post_test_tidyup () {
224# Save current exit status of caller script
225    RUN_STATUS=$? 
226    echo "Exit status: ${RUN_STATUS}" 
227#
228# requires the following variables defined and exported from the calling script:
229#  SETTE_DIR
230#  INPUT_DIR
231#  EXE_DIR
232#  CONFIG_DIR
233#  NEMO_VALIDATION_DIR
234#  NEW_CONF
235#  CMP_NAM
236#  TEST_NAME
237echo "SETTE directory is : ${SETTE_DIR}"
238echo "INPUT directory is : ${INPUT_DIR}"
239echo "EXECUTION directory is : ${EXE_DIR}"
240echo "CONFIG directory is : ${CONFIG_DIR}"
241echo "VALIDATION directory is : ${NEMO_VALID}"
242echo "NEW CONFIGURATION is : ${NEW_CONF}"
243echo "COMPILER is : ${CMP_NAM}"
244echo "TEST is : ${TEST_NAME}"
245echo "TOOLS directory is : ${TOOLS_DIR}"
246################################################################
247# SMALL DEBUG
248    EXIT_STATUS=${RUN_STATUS}
249    if [ ! -r ${EXE_DIR}/ocean.output ]
250        then
251        grep "E R R O R" ${EXE_DIR}/ocean.output && echo "Some ERRORS at execution time, see ${EXE_DIR}/ocean.output"
252        EXIT_STATUS=2
253        # exit 2 Error now catch in the report
254    fi
255
256    if [ ! -r ${EXE_DIR}/time.step ]
257        then
258        echo "file time.step does not exist"   >> ${SETTE_DIR}/output.sette
259        echo "some problems during execution of model"  >> ${SETTE_DIR}/output.sette
260        EXIT_STATUS=1
261        # exit 1 Error now catch in the report
262    else
263        echo "file time.step exists"  >> ${SETTE_DIR}/output.sette
264        echo "execution of model time step loop started"   >> ${SETTE_DIR}/output.sette
265    fi
266
267################################################################
268
269################################################################
270#
271# Creation of NEMO_VALIDATION tree
272#    set_valid_dir already done in sette_reference_config
273    mkdir -p ${NEMO_VALIDATION_DIR}
274    if [ -d ${NEMO_VALIDATION_DIR} ] ; then
275   echo "created ${NEMO_VALIDATION_DIR} directory"   >> ${SETTE_DIR}/output.sette
276    else
277   echo "problems in creating ${NEMO_VALIDATION_DIR} directory"   >> ${SETTE_DIR}/output.sette
278   echo "EXIT,"
279   exit 1
280    fi
281#
282# Exit before populating validation directory if the model run has
283# returned a non-zero exit status
284# On CRAY NEMO exit is not the expected 999 or 123456 (let this bloc in case useful later on).
285#    case ${EXIT_STATUS} in
286#        0|123456|999) echo " NEMO finished with exit code $EXIT_STATUS " ; post_test_tidyup ;;
287#        *) echo " NEMO abort on an unexpected error (segmentation fault or whatever) $EXIT_STATUS "
288#    esac
289
290    [ ${EXIT_STATUS} -ne 0 ] && exit ${EXIT_STATUS} 
291#
292# Save output & debug files in NEMO_VALIDATION tree
293    echo "saving ocean & ice output, run.stat, tracer.stat files ...." >> ${SETTE_DIR}/output.sette
294    echo "            " >> ${SETTE_DIR}/output.sette
295    [ -f ${EXE_DIR}/ocean.output ] && cp ${EXE_DIR}/*ocean.output ${NEMO_VALIDATION_DIR}/.
296    [ -f ${EXE_DIR}/run.stat ] && cp ${EXE_DIR}/*run.stat ${NEMO_VALIDATION_DIR}/.
297    [ -f ${EXE_DIR}/output.namelist.dyn ] && cp ${EXE_DIR}/*output.nam* ${NEMO_VALIDATION_DIR}/.
298    [ -f ${EXE_DIR}/tracer.stat ] && cp ${EXE_DIR}/*tracer.stat ${NEMO_VALIDATION_DIR}/.
299
300    if [ -n "$(ls ${NEMO_VALIDATION_DIR}/*run*)" ] ; then
301   echo "moved run.stat in ${NEMO_VALIDATION_DIR} directory"  >> ${SETTE_DIR}/output.sette
302   echo "moved run.stat in ${NEMO_VALIDATION_DIR} directory" 
303    else
304   echo "problem in looking for run.stat file in ${NEMO_VALIDATION_DIR} directory"  >> ${SETTE_DIR}/output.sette
305   echo "run.stat IS NOT in ${NEMO_VALIDATION_DIR} directory" 
306    fi
307    if [ -n "$(ls ${NEMO_VALIDATION_DIR}/*ocean.output*)" ] ; then
308   echo "moved ocean.output in ${NEMO_VALIDATION_DIR} directory"  >> ${SETTE_DIR}/output.sette
309   echo "moved ocean.output in ${NEMO_VALIDATION_DIR} directory" 
310    else
311   echo "problem in looking for ocean.output file in ${NEMO_VALIDATION_DIR} directory"  >> ${SETTE_DIR}/output.sette
312   echo "ocean.output IS NOT in ${NEMO_VALIDATION_DIR} directory" 
313    fi
314    if [ -n "$(ls ${NEMO_VALIDATION_DIR}/*tracer.stat*)" ] ; then
315        echo "moved tracer.stat in ${NEMO_VALIDATION_DIR} directory"  >> ${SETTE_DIR}/output.sette
316        echo "moved tracer.stat in ${NEMO_VALIDATION_DIR} directory"
317    else
318        echo "problem in looking for tracer.stat file in ${NEMO_VALIDATION_DIR} directory"  >> ${SETTE_DIR}/output.sette
319        echo "tracer.stat IS NOT in ${NEMO_VALIDATION_DIR} directory"
320    fi
321}
322
323#############################################################
324# extra functions to manipulate settings in the iodef.xml file
325#
326# Examples:
327#   set_xio_file_type    iodef.xml one_file
328#   set_xio_using_server iodef.xml true
329#   set_xio_buffer_size  iodef.xml 50000000
330#   set_xio_field_defs   iodef.xml
331#
332#############################################################
333
334usage2=" Usage : set_xio_file_type input_iodef.xml one_file||multiple_file"
335usage3=" Usage : set_xio_using_server input_iodef.xml true||false"
336usage4=" Usage : set_xio_buffer_size input_iodef.xml int_buffer_size"
337usage5=" Usage : set_xio_field_defs input_iodef.xml"
338
339set_xio_file_type () {
340        minargcount=2
341        if [ ${#} -lt ${minargcount} ]
342        then
343                echo "not enough arguments for set_xio_file_type"
344                echo "${usage2}"
345                exit 1
346        fi
347        if [ $2 != "one_file" ] && [ $2 != "multiple_file" ]
348        then
349                echo "unrecognised argument for set_xio_file_type"
350                echo "${usage2}"
351                echo $2
352                exit 1
353        fi
354        unset minargcount
355        if [  ! -f ${SETTE_DIR}/output.sette ] ; then
356                touch ${SETTE_DIR}/output.sette
357        fi
358
359        echo "executing script : set_xio_file_type $@" >> ${SETTE_DIR}/output.sette
360        echo "################" >> ${SETTE_DIR}/output.sette
361
362        VAR_NAME=$( grep "^.*<.*file_definition.*type.*=" ${EXE_DIR}/$1 | sed -e "s% *\!.*%%" )
363        if [ ${#VAR_NAME} -eq 0 ]
364        then
365                echo "doing \"set_xio_file_type $@\". "
366                echo "xml_tag: file_definition with variable: type is empty"
367                echo "confirm that an appropriate file_definition is in \"${EXE_DIR}/$1\" "
368                echo "exit"
369                echo "error in executing script : set_xio_file_type $@" >> ${SETTE_DIR}/output.sette
370                echo "....." >> ${SETTE_DIR}/output.sette
371                exit 1
372        fi
373        if [ $2 == "one_file" ] 
374        then
375           sed -e "s:multiple_file:one_file:" ${EXE_DIR}/$1 > ${EXE_DIR}/$1.tmp
376        else
377           sed -e "s:one_file:multiple_file:" ${EXE_DIR}/$1 > ${EXE_DIR}/$1.tmp
378        fi
379        mv ${EXE_DIR}/$1.tmp ${EXE_DIR}/$1
380
381        echo "finished script : set_xio_file_type $@" >> ${SETTE_DIR}/output.sette
382        echo "++++++++++++++++" >> ${SETTE_DIR}/output.sette
383        echo "                " >> ${SETTE_DIR}/output.sette
384}
385
386set_xio_using_server () {
387        minargcount=2
388        if [ ${#} -lt ${minargcount} ]
389        then
390                echo "not enough arguments for set_xio_using_server"
391                echo "${usage2}"
392                exit 1
393        fi
394        if [ $2 != "true" ] && [ $2 != "false" ]
395        then
396                echo "unrecognised argument for set_xio_using_server"
397                echo "${usage2}"
398                echo $2
399                exit 1
400        fi
401        unset minargcount
402        if [  ! -f ${SETTE_DIR}/output.sette ] ; then
403                touch ${SETTE_DIR}/output.sette
404        fi
405
406        echo "executing script : set_xio_using_server $@" >> ${SETTE_DIR}/output.sette
407        echo "################" >> ${SETTE_DIR}/output.sette
408
409        VAR_NAME=$( grep "^.*<.*variable id.*=.*using_server.*=.*bool" ${EXE_DIR}/$1 | sed -e "s% *\!.*%%" )
410        if [ ${#VAR_NAME} -eq 0 ]
411        then
412                echo "doing \"set_xio_using_server $@\". "
413                echo "xml_tag: "variable id=using_server" with variable: bool is empty"
414                echo "confirm that an appropriate variable id is in \"${EXE_DIR}/$1\" "
415                echo "exit"
416                echo "error in executing script : set_xio_using_server $@" >> ${SETTE_DIR}/output.sette
417                echo "....." >> ${SETTE_DIR}/output.sette
418                exit 1
419        fi
420        if [ $2 == "false" ]
421        then
422           sed -e "/using_server/s:true:false:" ${EXE_DIR}/$1 > ${EXE_DIR}/$1.tmp
423           export USING_MPMD=no
424        else
425           sed -e "/using_server/s:false:true:" ${EXE_DIR}/$1 > ${EXE_DIR}/$1.tmp
426           export USING_MPMD=yes
427        fi
428        mv ${EXE_DIR}/$1.tmp ${EXE_DIR}/$1
429
430        echo "finished script : set_xio_using_server $@" >> ${SETTE_DIR}/output.sette
431        echo "++++++++++++++++" >> ${SETTE_DIR}/output.sette
432        echo "                " >> ${SETTE_DIR}/output.sette
433}
434
435set_xio_buffer_size () {
436        minargcount=2
437        if [ ${#} -lt ${minargcount} ]
438        then
439                echo "not enough arguments for set_xio_buffer_size"
440                echo "${usage4}"
441                exit 1
442        fi
443        unset minargcount
444        if [  ! -f ${SETTE_DIR}/output.sette ] ; then
445                touch ${SETTE_DIR}/output.sette
446        fi
447
448        echo "executing script : set_xio_buffer_size $@" >> ${SETTE_DIR}/output.sette
449        echo "################" >> ${SETTE_DIR}/output.sette
450
451        VAR_NAME=$( grep "^.*<.*variable id.*=.*buffer_size.*=.*integer" ${EXE_DIR}/$1 | sed -e "s% *\!.*%%" )
452        if [ ${#VAR_NAME} -eq 0 ]
453        then
454                echo "doing \"set_xio_buffer_size $@\". "
455                echo "xml_tag: "variable id=buffer_size" with variable: integer is empty"
456                echo "confirm that an appropriate variable id is in \"${EXE_DIR}/$1\" "
457                echo "exit"
458                echo "error in executing script : set_xio_buffer_size $@" >> ${SETTE_DIR}/output.sette
459                echo "....." >> ${SETTE_DIR}/output.sette
460                exit 1
461        fi
462        sed -e "/buffer_size/s:>.*<:>$2<:" ${EXE_DIR}/$1 > ${EXE_DIR}/$1.tmp
463        mv ${EXE_DIR}/$1.tmp ${EXE_DIR}/$1
464
465        echo "finished script : set_xio_buffer_size $@" >> ${SETTE_DIR}/output.sette
466        echo "++++++++++++++++" >> ${SETTE_DIR}/output.sette
467        echo "                " >> ${SETTE_DIR}/output.sette
468}
469
470set_xio_field_defs () {
471        minargcount=1
472        if [ ${#} -lt ${minargcount} ]
473        then
474                echo "not enough arguments for set_xio_field_defs"
475                echo "${usage5}"
476                exit 1
477        fi
478        unset minargcount
479        if [  ! -f ${SETTE_DIR}/output.sette ] ; then
480                touch ${SETTE_DIR}/output.sette
481        fi
482
483        echo "executing script : set_xio_field_defs $@" >> ${SETTE_DIR}/output.sette
484        echo "################" >> ${SETTE_DIR}/output.sette
485
486
487        [ -f ${EXE_DIR}/field_def_nemo-oce.xml ] || sed -i '/field_def_nemo-oce/d' $1
488        [ -f ${EXE_DIR}/field_def_nemo-ice.xml ] || sed -i '/field_def_nemo-ice/d' $1
489        [ -f ${EXE_DIR}/field_def_nemo-pisces.xml ] || sed -i '/field_def_nemo-pisces/d' $1
490
491        echo "finished script : set_xio_field_defs $@" >> ${SETTE_DIR}/output.sette
492        echo "++++++++++++++++" >> ${SETTE_DIR}/output.sette
493        echo "                " >> ${SETTE_DIR}/output.sette
494}
Note: See TracBrowser for help on using the repository browser.