source: trunk/libIGCM/libIGCM_config/libIGCM_config.ksh @ 1486

Last change on this file since 1486 was 1486, checked in by jgipsl, 5 years ago
  • Added eval on ExeNameIn? after it has been controled that it is not equal "". This is done to enable variables in the executable name. For example gcm_${ResolAtm?}.nc
  • Initialized variables ResolAtm?, ResolOce?, OptMode? which can be used in the Executable section in config.card if they are set in the UserChoices? section in the same config.card.
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 54.9 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#===================================
15function IGCM_config_CommonConfiguration
16{
17  IGCM_debug_PushStack "IGCM_config_CommonConfiguration" $@
18
19  # Debug Print :
20  [ ${Verbosity} -gt 0 ] && echo
21  IGCM_debug_Print 1 "IGCM_config_CommonConfiguration" $@
22
23  # config.card path
24  configCardPath=$1
25
26  #==================================
27  typeset option auxprint
28
29  #==================================
30  # Read UserChoices section:
31  [ ${Verbosity} -gt 0 ] && echo
32  IGCM_debug_Print 1 "DefineArrayFromOption  : config_UserChoices"
33
34  IGCM_card_DefineArrayFromSection ${configCardPath} UserChoices
35  for option in ${config_UserChoices[*]} ; do
36    IGCM_card_DefineVariableFromOption ${configCardPath} UserChoices ${option}
37    eval auxprint=\${config_UserChoices_${option}}
38    IGCM_debug_Print 3 "${option} : ${auxprint}"
39  done
40
41  # Set variables with shorter names on some variables which 
42  # are used in the executable name in config.card
43  ResolAtm=${config_UserChoices_ResolAtm}
44  ResolOce=${config_UserChoices_ResolOce}
45  OptMode=${config_UserChoices_OptMode}
46
47  #==================================
48  # Read Ensemble section:
49  [ ${Verbosity} -gt 0 ] && echo
50  IGCM_debug_Print 1 "DefineArrayFromOption  : config_Ensemble"
51
52  IGCM_card_DefineArrayFromSection ${configCardPath} Ensemble
53  for option in ${config_Ensemble[*]} ; do
54    IGCM_card_DefineVariableFromOption ${configCardPath} Ensemble ${option}
55    eval auxprint=\${config_Ensemble_${option}}
56    IGCM_debug_Print 3 "${option} : ${auxprint}"
57  done
58
59  #==================================
60  # Read Post section:
61  [ ${Verbosity} -gt 0 ] && echo
62  IGCM_debug_Print 1 "DefineArrayFromOption : config_Post"
63
64  IGCM_card_DefineArrayFromSection ${configCardPath} Post
65  for option in ${config_Post[*]} ; do
66    IGCM_card_DefineVariableFromOption ${configCardPath} Post ${option}
67    eval auxprint=\${config_Post_${option}}
68    IGCM_debug_Print 3 "${option} : ${auxprint}"
69  done
70  [ ${Verbosity} -gt 0 ] && echo
71
72  #==================================
73  # Define default value to keep compatibility with previous card: means before changes due to TGCC
74  # Apply some overrules to ensure proper usage of computing centres resources
75  #
76  if [ X${PackDefault} = Xtrue ] ; then
77    if [ X${config_UserChoices_SpaceName} = XTEST ]; then
78      # TEST simulations will not be packed and will stay on SCRATCHDIR filesystem
79      IGCM_debug_Print 1 "SpaceName=TEST. OVERRULE PackFrequency to NONE"
80      config_Post_PackFrequency=NONE
81    else
82      # Default to RebuildFrequency if nothing has been set up related to PackFrequency
83      [ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
84    fi
85  else
86    # If we DO NOT apply pack in this computing center
87    config_Post_PackFrequency=NONE
88  fi
89
90  #====================================================
91  # Define ARCHIVE : Dedicated to large files
92  # Define STORAGE : Dedicated to small/medium files
93  # Define R_OUT   : Output tree located on ARCHIVE
94  # Define R_BUF   : Output tree located on STORAGE (files waiting treatment, or file lcoation when SpaceName=!PROD)
95  # Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
96  # Define R_TMP   : A temporary space used by IGCM_debug_send_AMQP_msg__MAILTUNNEL. Must be persistent in between jobs
97  IGCM_sys_defineArchives
98
99  #====================================================
100  # R_SAVE : Job output directory
101  # R_BUFR : Job output buffered directory
102  # R_CMIP : Job output directory for CMIP6 standard files
103
104  if ( [ ! X${config_UserChoices_SpaceName} = X ] && [ ! X${config_UserChoices_ExperimentName} = X ] ) ; then
105    FreeName=$( echo ${config_UserChoices_JobName} | sed 's/.*_//' )
106    if ( [ ! X${config_Ensemble_EnsembleName} = X ] && [ ! X${config_Ensemble_EnsembleDate} = X ] ) ; then
107      R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${FreeName}
108      R_FIGR=${R_FIG}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${FreeName}
109      R_BUFR=${R_BUF}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${FreeName}
110      R_DODS=${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${FreeName}
111      R_CMIP=${CMIP6_BUF}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${FreeName}
112    else
113      R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
114      R_FIGR=${R_FIG}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
115      R_BUFR=${R_BUF}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
116      R_DODS=${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
117      R_CMIP=${CMIP6_BUF}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName}
118    fi
119  else
120    if ( [ ! X${config_Ensemble_EnsembleName} = X ] && [ ! X${config_Ensemble_EnsembleDate} = X ] ) ; then
121      R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${config_UserChoices_JobName}
122      R_FIGR=${R_FIG}/${config_UserChoices_TagName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${config_UserChoices_JobName}
123      R_BUFR=${R_BUF}/${config_UserChoices_TagName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${config_UserChoices_JobName}
124      R_DODS=${config_UserChoices_TagName}/${config_Ensemble_EnsembleName}/${config_Ensemble_EnsembleDate}/${config_UserChoices_JobName}
125    else
126      R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
127      R_FIGR=${R_FIG}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
128      R_BUFR=${R_BUF}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
129      R_DODS=${config_UserChoices_TagName}/${config_UserChoices_JobName}
130    fi
131  fi
132
133  #====================================================
134  # Define R_OUT_KSH : Storage place for job output
135  # Define R_OUT_EXE : Storage place for binary used during simulation
136  R_OUT_KSH=${R_SAVE}/Out
137  R_OUT_EXE=${R_SAVE}/Exe
138
139  #====================================================
140  # Define R_BUF_KSH : Buffer place for job output
141  # Define R_BUF_EXE : Buffer place for binary used during simulation
142  R_BUF_KSH=${R_BUFR}/Out
143  R_BUF_EXE=${R_BUFR}/Exe
144
145  #====================================================
146  # Define REBUILD_DIR : where we store files needing rebuild process
147  REBUILD_DIR=${R_BUFR}/REBUILD
148  if [ ! X${TaskType} = Xchecking ] ; then
149    IGCM_sys_MkdirWork ${REBUILD_DIR}
150  fi
151
152  #====================================================
153  # DodsCopy : apply default value if not defined
154  if ( [ X${config_Post_DodsCopy} = X${NULL_STR} ] || [ X${config_Post_DodsCopy} = X ] ) ; then
155    config_Post_DodsCopy=TRUE
156  fi
157
158  #====================================================
159  # AtlasIPSL : apply default value if not defined
160  if ( [ X${config_Post_AtlasIPSL} = X${NULL_STR} ] || [ X${config_Post_AtlasIPSL} = X ] ) ; then
161    config_Post_AtlasIPSL=TRUE
162  fi
163
164  #====================================================
165  # ParserXIOS : apply default value if not defined
166  if ( [ X${config_Post_ParserXIOS} = X${NULL_STR} ] || [ X${config_Post_ParserXIOS} = X ] ) ; then
167    config_Post_ParserXIOS=FALSE
168  fi
169
170  #====================================================
171  # MetricsPCMDI : apply default value if not defined
172  if ( [ X${config_Post_MetricsPCMDI} = X${NULL_STR} ] || [ X${config_Post_MetricsPCMDI} = X ] ) ; then
173    config_Post_MetricsPCMDI=FALSE
174  fi
175
176  #====================================================
177  # IgnoreNonMonotonic : apply default value if not defined
178  if ( [ X${config_Post_IgnoreNonMonotonic} = X${NULL_STR} ] || [ X${config_Post_IgnoreNonMonotonic} = X ] ) ; then
179    config_Post_IgnoreNonMonotonic=FALSE
180  fi
181
182  #====================================================
183  # LightRestartPack : apply default value if not defined
184  if ( [ X${config_Post_LightRestartPack} = X${NULL_STR} ] || [ X${config_Post_LightRestartPack} = X ] ) ; then
185    config_Post_LightRestartPack=FALSE
186  fi
187
188  #====================================================
189  # Define StackFileLocation : directory where we store stack files
190  # Define StackFileName : stack file containing call tree and instrumentation
191  # Stack file containing call tree will be stored there.
192  if ( $DEBUG_debug ) ; then
193    StackFileLocation=${StackFileLocation:=${R_BUF_KSH}}
194    [ ! -d ${StackFileLocation} ] && mkdir -p ${StackFileLocation}
195    if [ X${TaskType} = Xcomputing ]; then
196      StackFileName=computing.stack.$$
197    elif [ X${TaskType} = Xpost-processing ]; then
198      StackFileName=${Script_Post_Output}.stack.$$
199    elif [ X${TaskType} = Xchecking ]; then
200      StackFileName=checking.stack.$$
201    else
202      IGCM_debug_Exit "IGCM_config_CommonConfiguration unknown TaskType : ${TaskType}"
203      IGCM_debug_Verif_Exit
204    fi
205
206    # This boolean will trigger the filling of the stack
207    # Only now we know where things should be ...
208    # We don't fill the stack when we perform checking task
209    if [ ! X${TaskType} = Xchecking ] ; then
210      ActivateStackFilling=true
211    fi
212  fi
213
214  IGCM_debug_PopStack "IGCM_config_CommonConfiguration"
215}
216
217#===================================
218function IGCM_config_Initialize
219{
220  IGCM_debug_PushStack "IGCM_config_Initialize"
221
222  # Debug Print :
223  echo
224  IGCM_debug_Print 1 "IGCM_config_Initialize"
225
226  # Test modipsl tree existence.
227  IGCM_sys_TestDir ${MODIPSL}
228  [ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDir"
229  IGCM_sys_TestDir ${libIGCM}
230  [ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDir"
231  IGCM_sys_TestDir ${R_EXE}
232  [ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDir"
233  IGCM_sys_TestDir ${SUBMIT_DIR}
234  [ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDir"
235
236  if ( $DEBUG_debug ) ; then
237    echo "Keep trace of inital SUBMIT_DIR : "
238    ls -lta ${SUBMIT_DIR}
239  fi
240
241  #==================================
242  # Read ListOfComponents section:
243  echo
244  IGCM_debug_Print 1 "DefineArrayFromSection : ListOfComponents"
245
246  IGCM_card_DefineArrayFromSection ${SUBMIT_DIR}/config.card ListOfComponents
247  for comp in ${config_ListOfComponents[*]} ; do
248    IGCM_card_DefineArrayFromOption ${SUBMIT_DIR}/config.card ListOfComponents ${comp}
249  done
250  IGCM_debug_Print 3 ${config_ListOfComponents[*]}
251
252  #==================================
253  # Read Executable section:
254  IGCM_card_DefineArrayFromSection   ${SUBMIT_DIR}/config.card Executable
255
256  #==================================
257  # Read Restarts section:
258  # Restarts : Gerneral rule or local for each component.
259  echo
260  IGCM_debug_Print 1 "DefineArrayFromOption : config_Restarts"
261
262  IGCM_card_DefineArrayFromSection ${SUBMIT_DIR}/config.card Restarts
263  for option in ${config_Restarts[*]} ; do
264    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/config.card Restarts ${option}
265    eval auxprint=\${config_Restarts_${option}}
266    IGCM_debug_Print 3 "${option} : ${auxprint}"
267  done
268
269  #==================================
270  # Define Job Outputs Name
271  echo
272  IGCM_debug_Print 2 "Define Script_Output_Prefix and Exe_Output"
273  Script_Output_Prefix=${config_UserChoices_Script_Output_Prefix:='Script_Output'}
274  IGCM_debug_Print 3 "Script_Output_Prefix = ${Script_Output_Prefix}"
275  Exe_Output=out_execution
276  IGCM_debug_Print 3 "Exe_Output           = ${Exe_Output}"
277
278  #===================================================================#
279  # Prepare variables available for ${COMP}.card and ${COMP}.driver   #
280  #             But available to any son functions                    #
281  #===================================================================#
282
283  # Convert yyyy-mm-dd date to gregorian yyyymmdd
284  DateBegin=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )
285  DateEnd=$(   IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd}   )
286
287  # Period Length In Days between DateBegin and DateEnd
288  (( ExperienceLengthInDays=$( IGCM_date_DaysBetweenGregorianDate ${DateEnd} ${DateBegin} )  + 1 ))
289  if [ ${ExperienceLengthInDays} -lt 0 ] ; then
290    IGCM_debug_Print 1 "Problem with dates in config.card : ${DateEnd} < ${DateBegin} ! You must check that."
291    IGCM_debug_Exit "IGCM_config_Initialize" " Wrong Dates."
292    IGCM_debug_Verif_Exit
293  fi
294
295  # Day and Year of Initial State (Given in julian format)
296  InitDay=$(( $( IGCM_date_ConvertGregorianDateToJulian $DateBegin ) % 1000 ))
297  InitYear=$(( $( IGCM_date_ConvertGregorianDateToJulian $DateBegin ) / 1000 ))
298
299  #================================================================#
300  #                  Test and Prepare directories                  #
301  #================================================================#
302
303  # ==> 4 kinds of input files :
304  #     1) R_INIT  : Initial State Files   (Etat0, carteveg)
305  #     2) R_BC    : Boundary Conditions   (Forcages, lai)
306  #     3) Parameters files (allready define through ${SUBMIT_DIR})
307  #     4) Restarts files   (allready define in IGCM_config_Initialize)
308
309  # Here we offer the possibility to redefine R_INIT, R_BC
310  # and PeriodNb through config.card
311  R_INIT=${config_UserChoices_R_INIT:=${R_IN}/INIT}
312  echo
313  IGCM_debug_Print 2 "(Re)Define R_INIT, R_BC and PeriodNb"
314  IGCM_debug_Print 3 "R_IN=${R_IN}"
315  IGCM_debug_Print 3 "R_INIT=${R_INIT}"
316  R_BC=${config_UserChoices_R_BC:=${R_IN}/BC}
317  IGCM_debug_Print 3  "R_BC=${R_BC}"
318  PeriodNb=${config_UserChoices_PeriodNb:=${PeriodNb}}
319  IGCM_debug_Print 3  "Loop in main Job with ${PeriodNb} period(s)"
320
321  # SD ADA SPECIFIC #
322  #      TO FIX     #
323  #IGCM_sys_TestDirArchive ${R_IN}
324  #[ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDirArchive"
325
326  if ( ${FirstInitialize} ) ; then
327    IGCM_sys_MkdirArchive   ${R_SAVE}
328    [ ! ${config_Post_PackFrequency} = NONE ] && IGCM_sys_Mkdir ${R_BUFR}
329  else
330    IGCM_sys_TestDirArchive ${R_SAVE}
331    [ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDirArchive ${R_SAVE}"
332
333    if [ ! ${config_Post_PackFrequency} = NONE ] ; then
334      IGCM_sys_TestDir ${R_BUFR}
335      [ $? != 0 ] && IGCM_debug_Exit "IGCM_sys_TestDir ${R_BUFR}"
336    fi
337
338    # Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
339    IGCM_config_StateCheck
340
341    # And EXIT if not OK
342    IGCM_debug_Verif_Exit
343  fi
344
345  #====================================================
346  # Experience type : DEB(ug), DEV(elopment), RUN
347  if [ X${JobType} != XRUN ] ; then
348    echo
349    echo "===================================================="
350    echo "libIGCM JOB is NOT in RUN type mode."
351    echo "!! OUTPUT files will NOT be PROTECTED !!"
352    echo "Be carefull : you can ERASE the result of this job !"
353
354    case ${JobType} in
355    DEB)
356      echo "DEBUG mode : activation of 'set -vx' mode."
357      echo "DEBUG mode : no protection for output files."
358      echo "DEBUG mode : if active force asynchronous rebuild frequency to PeriodLength frequency."
359      ;;
360    DEV)
361      echo "DEVelopment mode : no protection for output files."
362      echo "DEVelopment mode : if active force asynchronous rebuild frequency to PeriodLength frequency."
363      ;;
364    esac
365
366    if ( [ X${config_Post_RebuildFrequency} != XNONE ] && [ ${DRYRUN} -eq 0 ] ) ; then
367      if [ X${config_Post_RebuildFrequency} != X${config_UserChoices_PeriodLength} ] ; then
368        echo "------------"
369        echo "WARNING : Job is NOT in RUN mode then we will force REBUILD Frequency"
370        echo "          to PeriodLength : ${config_UserChoices_PeriodLength}"
371        echo "------------"
372        config_Post_RebuildFrequency=${config_UserChoices_PeriodLength}
373      fi
374    fi
375    echo "===================================================="
376    echo
377  fi
378
379  IGCM_debug_PopStack "IGCM_config_Initialize"
380}
381
382#===================================
383function IGCM_config_DaysInPeriodLength
384{
385  IGCM_debug_PushStack "IGCM_config_DaysInPeriodLength"
386
387  typeset i
388
389  # Determine number of day(s) in PeriodLength :
390  case ${config_UserChoices_PeriodLength} in
391  *Y|*y)
392    PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} | sed -e 's/[yY]//' )
393    echo
394    IGCM_debug_Print 2 "Number of years for PeriodLength : ${PeriodLengthInYears}"
395    PeriodLengthInDays=0
396    i=0
397    until [ $i -ge $PeriodLengthInYears ] ; do
398      (( PeriodLengthInDays = PeriodLengthInDays + $( IGCM_date_DaysInYear $(( year + i )) ) ))
399      (( i=i+1 ))
400    done
401    ;;
402  *M|*m)
403    PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} | sed -e 's/[mM]//' )
404    echo
405    IGCM_debug_Print 2 "Number of months for PeriodLength : ${PeriodLengthInMonths}"
406    PeriodLengthInDays=0
407    i=0
408    until [ $i -ge $PeriodLengthInMonths ] ; do
409      if [ $(( 10#${month} + ${i} )) -lt 13 ] ; then
410        (( PeriodLengthInDays  = PeriodLengthInDays + $( IGCM_date_DaysInMonth $year $(( 10#${month} + ${i} )) ) ))
411      else
412        (( PeriodLengthInDays  = PeriodLengthInDays + $( IGCM_date_DaysInMonth $year $(( 10#${month} + ${i} - 12 )) ) ))
413      fi
414      (( i=i+1 ))
415    done
416    ;;
417  *D|*d)
418    PeriodLengthInMonths=0
419    PeriodLengthInDays=$( echo ${config_UserChoices_PeriodLength} | sed -e 's/[dD]//' )
420    echo
421    IGCM_debug_Print 2 "Number of days for PeriodLength : ${PeriodLengthInDays}";;
422  *)
423    IGCM_debug_Exit "IGCM_config_DaysInPeriodLength " ${config_UserChoices_PeriodLength} " invalid period length : choose in *Y, *M, *D."
424    IGCM_debug_Verif_Exit ;;
425  esac
426
427  IGCM_debug_PopStack "IGCM_config_DaysInPeriodLength"
428}
429
430#===================================
431function IGCM_config_DateCoherency
432{
433  IGCM_debug_PushStack "IGCM_config_DateCoherency"
434
435  echo
436  IGCM_debug_Print 1 "IGCM_config_DateCoherency"
437  echo
438
439  typeset Length VerifiedPeriodDateBegin VerifiedPeriodDateEnd
440
441  # check coherency between (PeriodDateBegin, PeriodDateEnd) and (DateBegin, CumulPeriod, PeriodLength)
442  # DateBegin + CumulPeriod*PeriodLength = PeriodDateBegin
443  echo
444
445  case ${config_UserChoices_PeriodLength} in
446  *Y|*y)
447    Length=$( IGCM_date_DaysInCurrentPeriod ${DateBegin} $(( ${CumulPeriod} * ${PeriodLengthInYears} ))Y )
448    ;;
449  *M|*m)
450    Length=$( IGCM_date_DaysInCurrentPeriod ${DateBegin} $(( ${CumulPeriod} * ${PeriodLengthInMonths} ))M )
451    ;;
452  *D|*d)
453    Length=$( IGCM_date_DaysInCurrentPeriod ${DateBegin} $(( ${CumulPeriod} * ${PeriodLengthInDays} ))D )
454    ;;
455  esac
456  VerifiedPeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} ${Length}-1 )
457
458  if [ ${VerifiedPeriodDateEnd} != ${PeriodDateEnd} ] ; then
459    IGCM_debug_Print 1 "From run.card PeriodDateEnd is not consistent with DateBegin and CumulPeriod."
460    IGCM_debug_Print 1 "We have DateBegin = ${DateBegin}"
461    IGCM_debug_Print 1 "We have CumulPeriod = ${CumulPeriod}"
462    IGCM_debug_Print 1 "We have PeriodDateEnd = ${PeriodDateEnd}"
463    IGCM_debug_Print 1 "We have VerifiedPeriodDateEnd = ${VerifiedPeriodDateEnd}"
464    IGCM_debug_Print 1 "You must have change run.card in an inconsistent way."
465
466    IGCM_debug_Exit "STOP here to avoid further issues."
467  fi
468
469  # PeriodDateBegin + PeriodLength = PeriodDateEnd
470  VerifiedPeriodDateBegin=$( IGCM_date_AddDaysToGregorianDate ${VerifiedPeriodDateEnd} $(( ${PeriodLengthInDays} * -1 )) )
471
472  IGCM_debug_PopStack "IGCM_config_DateCoherency"
473}
474
475#===================================
476function IGCM_config_StateCheck
477{
478  IGCM_debug_PushStack "IGCM_config_StateCheck"
479
480    #Test state of run in run.card
481    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration PeriodState
482
483    if [ ${run_Configuration_PeriodState} = "Fatal" ] ; then
484      echo
485      IGCM_debug_Print 1 "!! Error in run.card with PeriodState : " ${run_Configuration_PeriodState} "!!"
486      IGCM_debug_Print 1 "Check the overall status of your simulation by visiting this page:"
487      IGCM_debug_Print 1 "http://prodiguer-test-web.ipsl.fr/static/simulation.detail.html?uid=${simuid}"
488      IGCM_debug_Print 1 "Then try running ${libIGCM}/clean_PeriodLenght.job to clean latest failed period"
489      IGCM_debug_Print 1 "Or try running ${libIGCM}/clean_latestPackperiod.job to clean lastest pack period."
490      IGCM_debug_Print 1 "Or use ${libIGCM}/purge_simulation.job to delete the full simulation."
491      IGCM_debug_Exit
492    elif [ $( echo ${run_Configuration_PeriodState} | grep Fatal | wc -l ) -eq 1 ] ; then
493      echo
494      IGCM_debug_Print 1 "!! Error in run.card with PeriodState : " ${run_Configuration_PeriodState} "!!"
495      IGCM_debug_Print 1 "Compute jobs has been stop because at least the above mentionned post-processing jobs fails."
496      IGCM_debug_Print 1 "Check post-processing jobs carefully by visiting this page:"
497      IGCM_debug_Print 1 "http://prodiguer-test-web.ipsl.fr/static/simulation.detail.html?uid=${simuid}"
498      IGCM_debug_Print 1 "Please visit that page to see how to fix issues:"
499      IGCM_debug_Print 1 "https://forge.ipsl.jussieu.fr/igcmg_doc/wiki/DocGmonitor"
500      IGCM_debug_Print 1 "Then try running ${libIGCM}/clean_PeriodLenght.job to clean latest failed period"
501      IGCM_debug_Print 1 "Or try running ${libIGCM}/clean_latestPackperiod.job to clean latest pack period."
502      IGCM_debug_Print 1 "Or use ${libIGCM}/purge_simulation.job to delete the full simulation."
503      IGCM_debug_Exit
504    fi
505
506  IGCM_debug_PopStack "IGCM_config_StateCheck"
507}
508
509#===================================
510function IGCM_config_Check
511{
512  IGCM_debug_PushStack "IGCM_config_Check"
513
514  # If one of the following modulo is not zero :
515  # we will issue an error then explain and exit in
516  # AA_job IGCM_debug_Verif_Exit call before binary submission
517
518  echo
519  IGCM_debug_Print 1 "IGCM_config_Check"
520  echo
521
522  typeset i
523
524  # Check RebuildFrequency against key frequencies : PeriodLength ; PackFrequency ; TimeSeriesFrequency ; SeasonalFrequency
525  if ( [ ! X${config_Post_RebuildFrequency} = X${NULL_STR} ] && [ ! X${config_Post_RebuildFrequency} = XNONE ] ) ; then
526    AsynchronousRebuild=true
527    IGCM_debug_Print 1 "Asynchronous rebuild has been activated."
528    echo
529    # modulo (RebuildFrequency and PeriodLength/TimeSeriesFrequency/SeasonalFrequency) must be zero
530    IGCM_debug_Print 1 "Check coherence between RebuildFrequency and PeriodLength"
531    IGCM_post_CheckModuloFrequency config_Post_RebuildFrequency config_UserChoices_PeriodLength
532    IGCM_debug_Print 1 "Check coherence between PackFrequency and RebuildFrequency"
533    IGCM_post_CheckModuloFrequency config_Post_PackFrequency config_Post_RebuildFrequency
534    IGCM_debug_Print 1 "Check coherence between TimeSeriesFrequency and RebuildFrequency"
535    IGCM_post_CheckModuloFrequency config_Post_TimeSeriesFrequency config_Post_RebuildFrequency
536    IGCM_debug_Print 1 "Check coherence between SeasonalFrequency and RebuildFrequency"
537    IGCM_post_CheckModuloFrequency config_Post_SeasonalFrequency config_Post_RebuildFrequency
538  else
539    AsynchronousRebuild=false
540    IGCM_debug_Print 1 "Asynchronous rebuild has not been activated"
541    IGCM_debug_Print 1 "Proceed with standard post-treatment pathway"
542    echo
543    #modulo (PeriodLength and TimeSeriesFrequency/SeasonalFrequency) must be zero
544    IGCM_debug_Print 1 "Check coherence between TimeSeriesFrequency and PeriodLength"
545    IGCM_post_CheckModuloFrequency config_Post_TimeSeriesFrequency config_UserChoices_PeriodLength
546    IGCM_debug_Print 1 "Check coherence between SeasonalFrequency and PeriodLength"
547    IGCM_post_CheckModuloFrequency config_Post_SeasonalFrequency   config_UserChoices_PeriodLength
548  fi
549
550  # Check PackFrequency against other key frequencies
551  # Modulo (PackFrequency and TimeSeriesFrequency/SeasonalFrequency and PeriodLenght) must be zero
552  if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
553    Pack=true
554    #
555    IGCM_debug_Print 1 "Check coherence between PackFrequency and PeriodLength"
556    IGCM_post_CheckModuloFrequency config_Post_PackFrequency config_UserChoices_PeriodLength
557    IGCM_debug_Print 1 "Check coherence between TimeSeriesFrequency and PackFrequency"
558    IGCM_post_CheckModuloFrequency config_Post_TimeSeriesFrequency config_Post_PackFrequency
559    IGCM_debug_Print 1 "Check coherence between SeasonalFrequency and PackFrequency"
560    IGCM_post_CheckModuloFrequency config_Post_SeasonalFrequency config_Post_PackFrequency
561  else
562    Pack=false
563  fi
564
565  # modulo (TimeSeriesFrequency and all Chunck2D) must be zero
566  NbJob=${#CHUNCK2D_SIZE[@]}
567  i=0
568  until [ $i -ge $NbJob ]; do
569    value=${CHUNCK2D_SIZE[${i}]}
570    IGCM_debug_Print 1 "Check coherence between ${CHUNCK2D_NAME[${i}]} Chunck2D frequency and TimeSeriesFrequency"
571    IGCM_post_CheckModuloFrequency value config_Post_TimeSeriesFrequency
572    case ${value} in
573    *Y|*y) ;;
574    *)
575      IGCM_debug_Print 1 "All ChunckJob2D frequency must be expressed in year *Y|*y in comp.card"
576      IGCM_debug_Exit "This will stop the job" ;;
577    esac
578    (( i=i+1 ))
579  done
580
581  # modulo (TimeSeriesFrequency and all Chunck3D) must be zero
582  NbJob=${#CHUNCK3D_SIZE[@]}
583  i=0
584  until [ $i -ge $NbJob ]; do
585    value=${CHUNCK3D_SIZE[${i}]}
586    IGCM_debug_Print 1 "Check coherence between ${CHUNCK3D_NAME[${i}]} Chunck3D frequency and TimeSeriesFrequency"
587    IGCM_post_CheckModuloFrequency value config_Post_TimeSeriesFrequency
588    case ${value} in
589    *Y|*y) ;;
590    *)
591      IGCM_debug_Print 1 "All ChunckJob3D frequency must be expressed in year *Y|*y in comp.card"
592      IGCM_debug_Exit "This will stop the job" ;;
593    esac
594    (( i=i+1 ))
595  done
596
597  # check to be sure there is enough space on temporary filesystems to run
598  echo
599  IGCM_debug_Print 1 "Check if there is enough space on temporary filesystem"
600  IGCM_sys_check_quota
601
602  # check to be sure that RUN_DIR_PATH, that will be removed is not pointing to an important directory
603  echo
604  IGCM_debug_Print 1 "Check where RUN_DIR_PATH variable is pointing to"
605  IGCM_sys_check_path
606
607
608  IGCM_debug_PopStack "IGCM_config_Check"
609}
610
611#===================================
612function IGCM_config_ConfigureExecution
613{
614  IGCM_debug_PushStack " IGCM_config_ConfigureExecution"
615
616  #echo
617  IGCM_debug_Print 1 " IGCM_config_ConfigureExecution"
618  #echo
619
620  typeset ExeNameIn ExeNameFirst CompNameFirst configCardPath comp i
621  typeset tempvar tempvarMPI tempvarNOD NbElts NbExec
622
623  # config.card path
624  configCardPath=$1
625
626  coreNumber=0
627  mpiTasks=0
628  openMPthreads=0
629  NbExec=0
630
631  OK_PARA_MPI=false
632  OK_PARA_OMP=false
633  OK_PARA_NOD=false
634  OK_PARA_MPMD=false
635
636  for comp in ${config_ListOfComponents[*]} ; do
637
638    # Manage component executable
639    IGCM_card_DefineArrayFromOption ${configCardPath} Executable ${comp}
640
641    eval ExeNameIn=\${config_Executable_${comp}[0]}
642
643    # NO order in config.card for parallelized values !
644    # just use suffix : MPI , OMP and NOD (for number of NODes.)
645
646    # NOD is the number of NODes allocated
647    eval ${comp}_PROC_NOD=0
648
649    # MPI is the number of MPI processus per nodes
650    eval ${comp}_PROC_MPI=0
651
652    # OMP is the number of OpenMP threads per MPI processus
653    eval ${comp}_PROC_OMP=0
654
655    # Only if we really have an executable for the component :
656    if ( [ "X${ExeNameIn}" != X\"\" ] && [ "X${ExeNameIn}" != "Xinca.dat" ] ) ; then
657
658      IGCM_debug_Print 1 ${comp}
659
660      # Keep the first executable found and the first CompName
661      eval ExeNameFirst=${ExeNameIn}
662      CompNameFirst=${comp}
663
664      # Are we a second executable?
665      (( NbExec = NbExec + 1 ))
666
667      # set 1 MPI task, 1 OpenMP thread and 1 node as default
668      eval ${comp}_PROC_MPI=1
669      eval ${comp}_PROC_OMP=1
670      eval ${comp}_PROC_NOD=1
671
672      eval NbElts=\${#config_Executable_${comp}[@]}
673
674      if [ ${NbElts} -gt 2 ] ; then
675        #
676        # CURRENT METHOD TO SPECIFY MPI AND OMP RESSOURCES
677        #
678        i=2
679        while [ ${i} -lt ${NbElts} ] ; do
680          eval tempvar=\${config_Executable_${comp}[${i}]}
681          IGCM_debug_Print 2 ${tempvar}
682
683          if [ X${tempvar} = X ] ; then
684            IGCM_debug_Print 2 "Error reading MPI/OMP parameters !!!"
685            IGCM_debug_Exit "Check your config.card. Exit now"
686            IGCM_debug_Verif_Exit
687          fi
688
689          case ${tempvar} in
690          *[mM][pP][iI]*)
691            # Read MPI parameter for composante
692            eval ${comp}_PROC_MPI=$( echo ${tempvar} | tr '[a-z]' '[A-Z]' | sed -e "s/MPI//" )
693            OK_PARA_MPI=true;;
694          *[oO][mM][pP]*)
695            # Read OMP parameter for composante
696            eval ${comp}_PROC_OMP=$( echo ${tempvar} | tr '[a-z]' '[A-Z]' | sed -e "s/OMP//" )
697            ;;
698          *[nN][oO][dD]*)
699            # Read NOD (NumBer of Nodes) parameter for composante
700            eval ${comp}_PROC_NOD=$( echo ${tempvar} | tr '[a-z]' '[A-Z]' | sed -e "s/NOD//" )
701            OK_PARA_NOD=true
702            OK_PARA_MPI=true
703            ;;
704          esac
705          (( i = i + 1 ))
706        done
707      else
708        #
709        # BACKWARD COMPATIBILITY NOT SUPPORTED ANYMORE
710        #
711        IGCM_debug_Exit "You are using a deprecated ressources specification mechanism in your config.card"
712        IGCM_debug_Exit "Please check : https://forge.ipsl.jussieu.fr/igcmg_doc/wiki/DocEsetup#ThesectionExecutable"
713        IGCM_debug_Exit "Please modify ${configCardPath}"
714        exit
715      fi
716      if [ ${comp}_PROC_NOD -gt 1 ] ; then
717            if ( [ ${comp}_PROC_MPI -gt 1 ] && [ ${comp}_PROC_OMP -gt 1 ] ) ; then
718              IGCM_debug_Print 2 "Error using MPI/OMP/NOD parameters !!!"
719              IGCM_debug_Exit "It is not allowed to use MPI and OMP and NOD parameters for the same component"
720              IGCM_debug_Exit "Check your config.card. Exit now"
721              IGCM_debug_Verif_Exit
722            fi
723            if [ "${comp}" != "IOS" ] ; then
724              IGCM_debug_Print 2 "Error using NOD parameter !!!"
725              IGCM_debug_Exit "NOD parameter is only available for IOS component "
726              IGCM_debug_Exit "Check your config.card. Exit now"
727              IGCM_debug_Verif_Exit
728            fi
729      fi
730
731      eval tempvarMPI=\${${comp}_PROC_MPI}
732      eval tempvarNOD=\${${comp}_PROC_NOD}
733      eval tempvarOMP=\${${comp}_PROC_OMP}
734
735      # set OMP mode if more than 1 OMP thread.
736      [ ${tempvarOMP} -ge 2 ] && OK_PARA_OMP=true
737
738      # Number of OMP threads
739      [ ${openMPthreads} -lt ${tempvarOMP} ] && openMPthreads=${tempvarOMP}
740
741      # SUM UP NUMBER OF CORES
742      if [ ${comp}_PROC_NOD -gt 1 ] ; then 
743          (( coreNumber = coreNumber + tempvarNOD * NB_CORE_PER_NODE ))
744      else
745          (( coreNumber = coreNumber + tempvarMPI * tempvarNOD * tempvarOMP ))
746      fi
747      # SUM UP NUMBER OF MPI TASKS
748      (( mpiTasks = mpiTasks + tempvarMPI * tempvarNOD ))
749    fi
750  done
751
752  # MANDATORY FOR THE OPA9.DRIVER. USED TO EDIT OPA NAMELIST
753  # WE SHOULD PLANIFY NUM_PROC_??? DEPRECATION
754  NUM_PROC_CPL=${CPL_PROC_MPI}
755  NUM_PROC_OCE=${OCE_PROC_MPI}
756  NUM_PROC_ATM=${ATM_PROC_MPI}
757
758  # set MPMD mode if more than 2 executable names.
759  [ ${NbExec} -ge 2 ] && OK_PARA_MPMD=true 
760
761  # Define the execution type we are running in
762  if ( ${OK_PARA_MPMD} ) ; then
763    if ( ${OK_PARA_MPI} ) ; then
764      # MPMD always implies MPI
765      executionType=1
766    fi
767    if ( ${OK_PARA_OMP} ) ; then
768      # MPMD + MPI/OMP
769      executionType=2
770    fi
771  else
772    if ( ( ${OK_PARA_MPI} ) && ( ${OK_PARA_OMP} ) ) ; then
773      # SPMD + MPI/OMP
774      executionType=3
775    elif ( ( ${OK_PARA_MPI} ) && ( ! ${OK_PARA_OMP} ) ) ; then
776      # SPMD + MPI only
777      executionType=4
778    elif ( ( ! ${OK_PARA_MPI} ) && ( ${OK_PARA_OMP} ) ) ; then
779      # SPMD + OMP only
780      executionType=5
781    elif ( ( ! ${OK_PARA_MPI} ) && ( ! ${OK_PARA_OMP} ) ) ; then
782      # SEQUENTIAL THEN
783      executionType=6
784      coreNumber=1
785    fi
786  fi
787
788  IGCM_debug_Print 1 "MPI/OMP treatment coreNumber = ${coreNumber}"
789  IGCM_debug_Print 1 "MPI/OMP treatment mpiTasks = ${mpiTasks}"
790  IGCM_debug_Print 1 "MPI/OMP treatment openMPthreads = ${openMPthreads}"
791  IGCM_debug_Print 1 "MPI/OMP treatment executionType = ${executionType}"
792
793  IGCM_debug_PopStack "IGCM_config_ConfigureExecution"
794}
795
796#===================================
797function IGCM_config_PeriodStart
798{
799  IGCM_debug_PushStack "IGCM_config_PeriodStart"
800
801  echo
802  IGCM_debug_Print 1 "IGCM_config_PeriodStart"
803  echo
804
805  if ( ${FirstInitialize} ) ; then
806    #================================================#
807    #         Initialize date/period information     #
808    #================================================#
809
810    IGCM_date_GetYearMonthDay ${DateBegin} year month day
811    IGCM_config_DaysInPeriodLength
812
813    PeriodDateBegin=${DateBegin}
814    PeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( ${PeriodLengthInDays} - 1 )) )
815    CumulPeriod=1
816
817    #=================================================#
818    #              Write updated run.card             #
819    #=================================================#
820
821    #Correct run.card Configuration for this period
822    IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodDateBegin ${PeriodDateBegin}
823    IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodDateEnd ${PeriodDateEnd}
824    IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration CumulPeriod ${CumulPeriod}
825    if [ X$( grep "SubmitPath" ${SUBMIT_DIR}/run.card ) != X ] ; then
826      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration SubmitPath ${SUBMIT_DIR}
827    fi
828
829    IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Running"
830
831  else
832    #================================================#
833    #         The file run.card allready exist       #
834    #================================================#
835
836    # Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
837    IGCM_config_StateCheck
838    # And EXIT if not OK
839    IGCM_debug_Verif_Exit
840
841    #===================================#
842    #        Read updated run.card      #
843    #===================================#
844
845    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration PeriodDateBegin
846    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration PeriodDateEnd
847    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration CumulPeriod
848
849    PeriodDateBegin=$( IGCM_date_ConvertFormatToGregorian ${run_Configuration_PeriodDateBegin} )
850    PeriodDateEnd=$( IGCM_date_ConvertFormatToGregorian ${run_Configuration_PeriodDateEnd} )
851    CumulPeriod=${run_Configuration_CumulPeriod}
852
853    LastPeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate $( IGCM_date_ConvertFormatToGregorian ${PeriodDateBegin} ) -1 )
854
855    if [ ${Period} = 1 ]; then
856      # save last Job output and current run.card
857      typeset Potential
858      IGCM_sys_Cd ${SUBMIT_DIR}
859      #
860      IGCM_debug_Print 2 "Save previous ksh job output"
861      for Potential in $( ls ${Script_Output_Prefix}_${config_UserChoices_JobName}.[0-9][0-9][0-9][0-9][0-9][0-9] ) ; do
862        if [ X${Pack} = Xtrue ] ; then
863          ( IGCM_sys_TestFileBuffer  ${R_BUF_KSH}/${Potential} ) || IGCM_sys_Cp ${Potential} ${R_BUF_KSH}/${Potential}.$$
864        else
865          ( IGCM_sys_TestFileArchive ${R_OUT_KSH}/${Potential} ) || IGCM_sys_Put_Out ${Potential} ${R_OUT_KSH}/${Potential}.$$ NOMOVE
866          IGCM_debug_Print 1 "Save of previous ksh job output is not done for Pack=false"
867        fi
868      done
869
870      #
871      IGCM_debug_Print 2 "Save current run.card"
872      IGCM_card_CheckConflict run.card
873
874      if [ X${Pack} = Xtrue ] ; then
875        IGCM_sys_Cp ${SUBMIT_DIR}/run.card ${R_BUF_KSH}/run.card
876      else
877        IGCM_sys_Put_Out ${SUBMIT_DIR}/run.card ${R_OUT_KSH}/run.card NOMOVE
878        IGCM_debug_Print 1 "Save of run.card is not done for Pack=false"
879      fi
880      #
881      IGCM_sys_Cd ${RUN_DIR}
882    else
883      unset FileToBeDeleted
884    fi
885
886    # Determine number of day(s) in PeriodLength
887    IGCM_date_GetYearMonthDay $PeriodDateBegin year month day
888    IGCM_config_DaysInPeriodLength
889
890    # Check coherency between (PeriodDateBegin, PeriodDateEnd) and (DateBegin, CumulPeriod, PeriodLength)
891    IGCM_config_DateCoherency
892    # And EXIT if not OK
893    IGCM_debug_Verif_Exit
894
895    # Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
896    IGCM_config_StateCheck
897    # And EXIT if not OK
898    IGCM_debug_Verif_Exit
899
900    # We can say we are "Running" now.
901    IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Running"
902  fi
903
904  # BEGIN: SHOULD GO IN A FUNCTION FROM libIGCM_date.ksh
905  # Compute year_m1 and year_p1 (year minus 1Y and year plus 1Y)
906  year_m1=$(( year - 1 ))
907  year_p1=$(( year + 1 ))
908  # Compute month_m1 (month minus 1M)
909  # Compute yyyymm_m1 (yyyymm minus 1M)
910  month_m1=$(( 10#${month} - 1 ))
911  if [ ${month_m1} = 0 ]; then
912    month_m1=12
913    yyyymm_m1=${year_m1}12
914  elif [ ${month_m1} -le 9 ]; then
915    month_m1=0${month_m1}
916    yyyymm_m1=${year}${month_m1}
917  else
918    yyyymm_m1=${year}${month_m1}
919  fi
920  # Compute month_p1 (month plus 1M)
921  # Compute yyyymm_p1 (yyyymm plus 1M)
922  month_p1=$(( 10#${month} + 1 ))
923  if [ ${month_p1} = 13 ]; then
924    month_p1=01
925    yyyymm_p1=${year_p1}01
926  elif [ ${month_p1} -le 9 ]; then
927    month_p1=0${month_p1}
928    yyyymm_p1=${year}${month_p1}
929  else
930    yyyymm_p1=${year}${month_p1}
931  fi
932
933  #===================================================================#
934  # Calculate CyclicYear to be used for looping over a given forcing  #
935  # period. Add CyclicBegin and CyclicEnd in config.card UserChoices. #
936  #===================================================================#
937
938  # To use the variable CyclicYear, one must add in config.card CyclicBegin and CyclicEnd.
939  # CyclicBegin is the first year in the cycle. CyclicEnd is the last year included in the cycle.
940  if ( [ ! X${config_UserChoices_CyclicBegin} = X ] && [ ! X${config_UserChoices_CyclicEnd} = X ] ) ; then
941    CycleNb=$(( ${config_UserChoices_CyclicEnd} - ${config_UserChoices_CyclicBegin} + 1 ))
942    CyclicYear_p1=NOTDEFINED
943
944    # For current year
945    yeartmp=$year
946    diffy=$(( $yeartmp - ${config_UserChoices_CyclicBegin} ))
947    while [ $diffy -lt 0 ] ; do
948      yeartmp=$(( ${yeartmp} + ${CycleNb} ))
949      diffy=$(( $yeartmp - ${config_UserChoices_CyclicBegin} ))
950    done
951    CyclicYear=$(( ( ${diffy} % ${CycleNb} ) + ${config_UserChoices_CyclicBegin} ))
952
953    # For next coming year
954    yeartmp=$(( $year + 1 ))
955    diffy=$(( $yeartmp - ${config_UserChoices_CyclicBegin} ))
956    while [ $diffy -lt 0 ] ; do
957      yeartmp=$(( ${yeartmp} + ${CycleNb} ))
958      diffy=$(( $yeartmp - ${config_UserChoices_CyclicBegin} ))
959    done
960    CyclicYear_p1=$(( ( ${diffy} % ${CycleNb} ) + ${config_UserChoices_CyclicBegin} ))
961
962    IGCM_debug_Print 1 "CyclicYear   = ${CyclicYear}, CyclicYear_p1 = ${CyclicYear_p1}, current year=$year"
963  else
964    CyclicYear="ERROR_CyclicYear_Variable_Not_Defined"
965    CyclicYear_p1="ERROR_CyclicYear_p1_Variable_Not_Defined"
966    IGCM_debug_Print 1 "CyclicYear wont be use without adding CyclicBegin and CyclicEnd in config.card"
967  fi
968
969  # END: SHOULD GO IN A FUNCTION FROM libIGCM_date.ksh
970
971  #===================================================================#
972  # Prepare variables available for ${COMP}.card and ${COMP}.driver   #
973  #             But available to any son functions                    #
974  #===================================================================#
975
976  # Period Length In Days between DateBegin and DateCurrent (at end of period == PeriodDateEnd !)
977  (( SimulationLengthInDays = $( IGCM_date_DaysBetweenGregorianDate ${PeriodDateEnd} ${DateBegin} ) + 1 ))
978
979  # Debug Print :
980  echo
981  IGCM_debug_Print 1 "IGCM_config_PeriodStart : Before Execution"
982  IGCM_debug_Print 1 "Year of simulation      : ${year}"
983  IGCM_debug_Print 1 "Month of simulation     : ${month}"
984  IGCM_debug_Print 1 "PeriodLengthInDays      : ${PeriodLengthInDays}"
985  IGCM_debug_Print 1 "PeriodDateBegin         : ${PeriodDateBegin}"
986  IGCM_debug_Print 1 "PeriodDateEnd           : ${PeriodDateEnd}"
987  IGCM_debug_Print 1 "SimulationLengthInDays  : ${SimulationLengthInDays}"
988  IGCM_debug_Print 1 "ExperienceLengthInDays  : ${ExperienceLengthInDays}"
989
990  #================================================================#
991  #         Prepare variables available for comp_finalyze          #
992  #================================================================#
993
994  # Period for save files
995  DatesPeriod=${PeriodDateBegin}_${PeriodDateEnd}
996
997  # Prefix for save files of this period
998  PREFIX=${config_UserChoices_JobName}_${DatesPeriod}
999
1000  # List of files that will be deleted in RUN_DIR after run
1001  [ -f stack ] && FileToBeDeleted[0]="stack"
1002
1003  # Test if the same run as already been saved :
1004  if [ X${JobType} = XRUN ] ; then
1005    if [ ${DRYRUN} -le 0 ] ; then
1006      if ( IGCM_sys_TestFileBuffer ${R_BUF_KSH}/${PREFIX}_${Exe_Output} ) ; then
1007        IGCM_debug_Exit "IGCM_config_PeriodStart" "You are currently re-running an old job."
1008        IGCM_debug_Print 1 "Because of readonly permissions, you can't re-run a job when saved files"
1009        IGCM_debug_Print 1 " are still in the ARCHIVE directory. You must deleted those files, or "
1010        IGCM_debug_Print 1 " the whole ${R_SAVE} tree. See different clean_*.job in ${libIGCM} directory."
1011        IGCM_debug_Print 1 " This exit has been initiated because at least ${R_BUF_KSH}/${PREFIX}_${Exe_Output} exists."
1012        IGCM_debug_Verif_Exit
1013      fi
1014    fi
1015  else
1016    if ( IGCM_sys_TestFileBuffer ${R_BUF_KSH}/${PREFIX}_${Exe_Output} ) ; then
1017      IGCM_debug_Print 1 "IGCM_config_PeriodStart" "RErun an old job. Allowed in DEBUG or DEV mode."
1018    fi
1019  fi
1020
1021  #================================================================#
1022  #       Prepare variables available for binary execution         #
1023  #       Call function for creation of run script                 #
1024  #       Only done once per job                                   #
1025  #================================================================#
1026
1027  if [ ${Period} -eq 1 ]; then
1028    # Define the execution context (MPMD, SPMD, MPI/OMP ...)
1029    IGCM_config_ConfigureExecution ${SUBMIT_DIR}/config.card
1030    # Create the execution script for the current context
1031    IGCM_sys_build_execution_scripts
1032  fi
1033
1034  ExecutionFail=false
1035
1036  # Update the rabbitMQ queue
1037  IGCM_debug_BigBro_Update
1038
1039  IGCM_debug_PopStack "IGCM_config_PeriodStart"
1040}
1041
1042#===================================
1043function IGCM_config_SaveSourceModifications
1044{
1045  IGCM_debug_PushStack "IGCM_config_SaveSourceModifications"
1046
1047  typeset ExeOutDateMax listVarEnv
1048  ExeOutDateMax=$1
1049
1050  listVarEnv="ExeOutDateMax,R_OUT_EXE,PREFIX,SUBMIT_DIR"
1051  IGCM_sys_RshMaster "\
1052    . ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh; \
1053    export ExeOutDateMax=${ExeOutDateMax};\
1054    export R_OUT_EXE=${R_OUT_EXE};\
1055    export PREFIX=${PREFIX};\
1056    export SUBMIT_DIR=${SUBMIT_DIR};\
1057    export listVarEnv=${listVarEnv};\
1058    Script_Output=out_SaveSourceModifications;\
1059    IGCM_sys_Qsub ${libIGCM}/SaveSourceModifications.job ${ExeOutDateMax} ${R_OUT_EXE} ${PREFIX} ${SUBMIT_DIR}"
1060
1061  IGCM_debug_PopStack "IGCM_config_SaveSourceModifications"
1062}
1063
1064#===================================
1065function IGCM_config_PeriodEnd
1066{
1067  IGCM_debug_PushStack "IGCM_config_PeriodEnd"
1068
1069  echo
1070  IGCM_debug_Print 1 "IGCM_config_PeriodEnd"
1071  echo
1072
1073  if [ ${DRYRUN} -le 1 ] ; then
1074
1075    IGCM_debug_Print 1 "Check components binary : size and creation date"
1076
1077    typeset LS_comp LS_bin ExeDate ExeCpuLog NextExeSize LastCompExeSize
1078    typeset comp i
1079    typeset ExeNameIn ExeNameOut UpdateExe ExeSecDateMax
1080
1081    #==================================#
1082    #        Get last Exe Size         #
1083    #==================================#
1084
1085    (( i=0 ))
1086    if ( ${FirstInitialize} ) ; then
1087      run_Log_LastExeSize=""
1088      for comp in ${config_ListOfComponents[*]} ; do
1089        run_Log_LastExeSize[$i]=0
1090        (( i=i+1 ))
1091      done
1092    else
1093      IGCM_card_DefineArrayFromOption ${SUBMIT_DIR}/run.card Log LastExeSize
1094    fi
1095    #==================================#
1096    #         And Build ExeDate        #
1097    #==================================#
1098
1099    # ExeDate = ATM_Jun_12_09:34-SRF_Jun_12_09:34-OCE_Jun_12_09:34-ICE_Jun_12_09:34-CPL_Jun_12_09:33
1100    # Would be nice to have next line but no way to format ls output (need to ls -l --time-style "+%Y-%m-%dT%H:%M")
1101    # ExeDate = ATM_2009-06-12T09:34+SRF_2009-06-12T09:34+OCE_2009-06-12T09:34+ICE_2009-06-12T09:34+CPL_2009-06-12T09:34
1102    ExeDate=""
1103    NextExeSize="( "
1104    (( i=0 ))
1105    UpdateExe=false
1106    (( ExeSecDateMax = 0 ))
1107    for comp in ${config_ListOfComponents[*]} ; do
1108
1109      IGCM_debug_Print 3 ${comp}
1110
1111      eval ExeNameIn=\${config_Executable_${comp}[0]}
1112      eval ExeNameOut=\${config_Executable_${comp}[1]}
1113      # Only if we really have an executable for the component :
1114      if [ X${ExeNameIn} = X\"\" ] ; then
1115        # If there is no exe file for this component
1116        (( ExeSize=0 ))
1117      else
1118        eval ExeNameIn=${ExeNameIn}
1119        LS_bin=${R_EXE}/${ExeNameIn}
1120        IGCM_sys_FileSize ${LS_bin} ExeSize
1121
1122        set +A LS_comp -- $( LC_TIME=en_US ls -l ${LS_bin} )
1123        if [ X${ExeDate} = X ] ; then
1124          # First component exe date
1125          ExeDate=${comp}_${LS_comp[5]}_${LS_comp[6]}
1126        else
1127          ExeDate=${ExeDate}-${comp}_${LS_comp[5]}_${LS_comp[6]}
1128        fi
1129        ExeDate=${ExeDate}_${LS_comp[7]}
1130      fi
1131
1132      if [ ${i} -eq 0 ] ; then
1133        # First component
1134        NextExeSize="( "${ExeSize}
1135      else
1136        NextExeSize=${NextExeSize}", "${ExeSize}
1137      fi
1138      LastCompExeSize=${run_Log_LastExeSize[$i]}
1139      (( i=i+1 ))
1140
1141      if [ ${ExeSize} -ne ${LastCompExeSize} ] ; then
1142        if ( ${FirstInitialize} ) ; then
1143          IGCM_debug_Print 1 "Save first ${ExeNameIn} in ${R_OUT_EXE} !"
1144        else
1145          IGCM_debug_Print 1 "${ExeNameIn} has changed in ${R_EXE} !"
1146          IGCM_debug_Print 1 "Save latest ${ExeNameIn} in ${R_OUT_EXE} !"
1147          FileToBeDeleted[${#FileToBeDeleted[@]}]=${ExeNameOut}
1148        fi
1149        IGCM_sys_Put_Out ${ExeNameOut} ${R_OUT_EXE}/${PREFIX}_${ExeNameIn} rw
1150        UpdateExe=true
1151
1152        # SD : switch off for now
1153        #IGCM_sys_GetDate_FichWork ${LS_bin} ExeSecDate
1154        #if [ $ExeSecDateMax -lt $ExeSecDate ] ; then
1155        #  ExeSecDateMax=$ExeSecDate
1156        #fi
1157      fi
1158    done
1159
1160    # SD : switch off for now
1161    #if ( ${UpdateExe} ) ; then
1162    #  echo "Launch SaveSourceModifications."
1163    #  IGCM_config_SaveSourceModifications ${ExeSecDateMax}
1164    #fi
1165
1166    NextExeSize=${NextExeSize}" )"
1167    IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Log LastExeSize "${NextExeSize}"
1168
1169    if [ ${DRYRUN} -le 1 ] ; then
1170      tail -1500 ${Exe_Output} > ${Exe_Output}_tail.txt
1171      ExeCpuLog=$( gawk -f ${libIGCM}/libIGCM_sys/IGCM_add_out.awk ${Exe_Output}_tail.txt )
1172      RET=$?
1173      if [ $RET -eq 0 ] ; then
1174        # ExeCpuLog variable contents 5 fields
1175        echo "${CumulPeriod} ${PeriodDateBegin} ${PeriodDateEnd} ${ExeCpuLog} ${ExeDate}" |   \
1176          gawk '{printf("# %11d | %15s | %15s | %19s | %19s | %15.5f | %15.5f | %15.5f | %s\n", \
1177            $1,$2,$3,$4,$5,$6,$7,$8,$9)}' >> ${SUBMIT_DIR}/run.card
1178      fi
1179      FileToBeDeleted[${#FileToBeDeleted[@]}]=${Exe_Output}_tail.txt
1180    fi
1181
1182  fi
1183
1184  #==================================#
1185  #         Save Job output          #
1186  #==================================#
1187  if [ X${Pack} = Xtrue ] ; then
1188    IGCM_sys_PutBuffer_Out ${Exe_Output} ${R_BUF_KSH}/${PREFIX}_${Exe_Output}
1189  else
1190    IGCM_sys_Put_Out ${Exe_Output} ${R_OUT_KSH}/${PREFIX}_${Exe_Output}
1191  fi
1192  FileToBeDeleted[${#FileToBeDeleted[@]}]=${Exe_Output}
1193
1194  # All was right ? no ? then we stop.
1195  IGCM_debug_Verif_Exit
1196
1197  # If all was OK, we can delete all files not necessary for next Job
1198  echo
1199  IGCM_debug_Print 1 "Files that will be deleted before next period-run : "
1200
1201  if [ ${DRYRUN} -le 2 ] ; then
1202    for f in ${FileToBeDeleted[@]} ; do [ -f ${f} ] && ls -la $f ; [ -f ${f} ] && rm -f $f ; done
1203  else
1204    echo ${FileToBeDeleted[@]}
1205  fi
1206
1207  # Send some accounting element to the user if CumulPeriod=3
1208  if [ ${CumulPeriod} -eq 3 ] ; then
1209    echo
1210    IGCM_debug_Print 1 "Send email containing some accounting information : "
1211
1212    RealCpuTime=$( echo ${ExeCpuLog} | gawk '{print $3}' )
1213
1214    consumeHoursPerPeriod=$( echo "scale=6;${RealCpuTime}*${coreNumber}/3600" | bc )
1215
1216    consumeHoursPerWholeSimulation=$( echo "scale=6;${consumeHoursPerPeriod}/${PeriodLengthInDays}*${ExperienceLengthInDays}" | bc )
1217
1218    recommendedPeriodNb=$( echo "scale=6;${jobWarningDelay}/3600/${consumeHoursPerPeriod}*${coreNumber}" | bc )
1219
1220    IGCM_sys_SendMail Accounting
1221  fi
1222
1223  #=================================================#
1224  #         Modification of libIGCM behaviour       #
1225  #=================================================#
1226
1227  # To use this function, one must copy libIGCM.card from ${libIGCM} directory
1228  # and put it in ${SUBMIT_DIR} directory. After modifications of ${SUBMIT_DIR}/libIGCM.card,
1229  # variables define inside [UserChanges] will be modified for next Period of libIGCM main loop.
1230  if [ -f ${SUBMIT_DIR}/libIGCM.card ] ; then
1231    echo
1232    echo "########################################################################"
1233    echo "!!!                 Modification of libIGCM behaviour                !!!"
1234    echo
1235
1236    IGCM_debug_Print 1 "DefineArrayFromOption  : libIGCM_UserChanges in libIGCM.card"
1237    IGCM_card_DefineArrayFromSection ${SUBMIT_DIR}/libIGCM.card UserChanges
1238    IGCM_debug_Print 2 "libIGCM_UserChanges" ${libIGCM_UserChanges[*]}
1239
1240    # Special treatments for libIGCM internals
1241    for option in ${libIGCM_UserChanges[*]} ; do
1242      IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/libIGCM.card UserChanges ${option}
1243
1244      echo "We will change : ${option}."
1245      eval echo "Previous value : " \${${option}}
1246      eval echo "Change to : " \${libIGCM_UserChanges_${option}}
1247
1248      eval ${option}=\${libIGCM_UserChanges_${option}}
1249
1250      case ${option} in
1251      config_UserChoices_DateEnd)
1252        IGCM_debug_PrintVariables 1 config_UserChoices_DateEnd
1253        DateEnd=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )
1254
1255        # Period Length In Days between DateBegin and DateEnd
1256        (( ExperienceLengthInDays=$( IGCM_date_DaysBetweenGregorianDate ${DateEnd} ${DateBegin} )  + 1 ))
1257        if [ ${ExperienceLengthInDays} -lt 0 ] ; then
1258          IGCM_debug_Print 1 "Problem with dates in libIGCM.card : ${DateEnd} < ${DateBegin} ! You must check that."
1259          IGCM_debug_Exit "IGCM_PeriodEnd have wrong dates."
1260          IGCM_debug_Verif_Exit
1261        fi
1262        ;;
1263      config_UserChoices_PeriodLength)
1264        IGCM_debug_Print 1  "Change config_UserChoices_PeriodLength=${config_UserChoices_PeriodLength}"
1265        ;;
1266      PeriodNb)
1267        IGCM_debug_Print 1  "Loop in main Job with ${PeriodNb} period(s)"
1268        ;;
1269      config_Post_RebuildFrequency)
1270        IGCM_debug_Print 1  "Change config_Post_RebuildFrequency=${config_Post_RebuildFrequency} : IGCM_post_Configure"
1271        IGCM_post_Configure
1272        ;;
1273      config_Post_TimeSeriesFrequency)
1274        IGCM_debug_Print 1  "Change config_Post_TimeSeriesFrequency = ${config_Post_TimeSeriesFrequency} : IGCM_post_Configure"
1275        IGCM_post_Configure
1276        ;;
1277      config_Post_SeasonalFrequency)
1278        IGCM_debug_Print 1  "Change config_Post_SeasonalFrequency = ${config_Post_SeasonalFrequency} : IGCM_post_Configure"
1279        IGCM_post_Configure
1280        ;;
1281      esac
1282    done
1283
1284    echo
1285    echo "########################################################################"
1286    echo
1287  fi
1288
1289  #=================================================#
1290  #         Determine next computed period          #
1291  #=================================================#
1292
1293  PeriodDateBegin=$( IGCM_date_AddDaysToGregorianDate ${PeriodDateEnd} 1 )
1294  IGCM_date_GetYearMonthDay $PeriodDateBegin year month day
1295  year_m1=$(( year - 1 ))
1296  year_p1=$(( year + 1 ))
1297  IGCM_config_DaysInPeriodLength
1298  PeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate ${PeriodDateBegin} $(( ${PeriodLengthInDays} - 1 )) )
1299
1300  # Debug Print :
1301  echo
1302  IGCM_debug_Print 1 "IGCM_config_PeriodEnd : Preparing Next Execution"
1303  IGCM_debug_Print 1 "PeriodDateBegin       : ${PeriodDateBegin}"
1304  IGCM_debug_Print 1 "PeriodDateEnd         : ${PeriodDateEnd}"
1305  IGCM_debug_Print 1 "PeriodLengthInDays    : ${PeriodLengthInDays}"
1306
1307  PeriodDateBegin=$( IGCM_date_ConvertFormatToHuman ${PeriodDateBegin} )
1308  PeriodDateEnd=$( IGCM_date_ConvertFormatToHuman ${PeriodDateEnd} )
1309
1310  (( CumulPeriod = CumulPeriod + 1 ))
1311
1312  # Debug Print :
1313  echo
1314  IGCM_debug_Print 3 "PeriodDateBegin Human : ${PeriodDateBegin}"
1315  IGCM_debug_Print 3 "PeriodDateEnd Human   : ${PeriodDateEnd}"
1316  IGCM_debug_Print 3 "CumulPeriod           : ${CumulPeriod}"
1317
1318  #=================================================#
1319  #             Write updated run.card              #
1320  #=================================================#
1321
1322  IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodDateBegin ${PeriodDateBegin}
1323  IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodDateEnd ${PeriodDateEnd}
1324  IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration CumulPeriod ${CumulPeriod}
1325
1326  if ( ${FirstInitialize} ) ; then
1327    # It's no more the first time
1328    FirstInitialize=false
1329  fi
1330
1331  IGCM_debug_PopStack "IGCM_config_PeriodEnd"
1332}
1333
1334#===================================
1335function IGCM_config_Finalize
1336{
1337  IGCM_debug_PushStack "IGCM_config_Finalize"
1338
1339  echo
1340  IGCM_debug_Print 1 "IGCM_config_Finalize"
1341  echo
1342
1343  # Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
1344  IGCM_config_StateCheck
1345
1346  # And EXIT if not OK
1347  IGCM_debug_Verif_Exit
1348
1349  if [ ${SimulationLengthInDays} -ge ${ExperienceLengthInDays} ] ; then
1350    #==========================#
1351    # End of entire simulation #
1352    #==========================#
1353    simulationIsOver=true
1354
1355    # Mail notification
1356    IGCM_sys_SendMail
1357    #
1358    IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Completed"
1359    #
1360    IGCM_debug_Print 1 "Normal End of computation."
1361
1362  else
1363    #=================#
1364    # Submit next job #
1365    #=================#
1366    simulationIsOver=false
1367
1368    IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "OnQueue"
1369
1370    # Name of next Ksh Script output :
1371    Script_Output=${Script_Output_Prefix}_${config_UserChoices_JobName}.$( printf "%06d" ${CumulPeriod} )
1372
1373    IGCM_debug_Print 1 "Submit next job"
1374    # SUBMIT NEXT JOB from SUBMIT_DIR and come back in RUN_DIR
1375    IGCM_sys_Cd ${SUBMIT_DIR}
1376    # Keep only the 5 latest ${Script_Output_Prefix}_${config_UserChoices_JobName}
1377    ScriptTot=$( ls ${Script_Output_Prefix}_${config_UserChoices_JobName}.?????? 2>/dev/null | wc -l )
1378    [ ${ScriptTot} -gt 5 ] && rm -f $( ls ${Script_Output_Prefix}_${config_UserChoices_JobName}.?????? | head -$(( ${ScriptTot} - 5 )) )
1379    # Submit next job and come back
1380    IGCM_sys_Qsub ${SUBMIT_DIR}/Job_${config_UserChoices_JobName}
1381    IGCM_sys_Cd -
1382  fi
1383
1384  # Clean ${RUN_DIR}=${RUN_DIR_PATH}/${config_UserChoices_JobName}.${$}
1385  # Only for production run (No clean up in DEV or DEB mode)
1386  # and command sent from .. directory.
1387  IGCM_sys_Cd ..
1388  [ X${JobType} = XRUN ] && IGCM_sys_RmRunDir -rf ${RUN_DIR_PATH}
1389
1390  # Inform the rabbitMQ queue
1391  IGCM_debug_BigBro_Finalize
1392
1393  IGCM_debug_PopStack "IGCM_config_Finalize"
1394}
1395
1396#===================================
Note: See TracBrowser for help on using the repository browser.