source: CMIP6/ScenarioMIP/CM61-LR-scen-ssp245-r11/Job_CM61-LR-scen-ssp245-r11 @ 4425

Last change on this file since 4425 was 4425, checked in by devillim, 5 years ago
File size: 19.9 KB
Line 
1#!/bin/ksh
2######################
3## IRENE   TGCC/CEA ##
4######################
5#MSUB -r CM61-LR-scen-ssp245-r11        # Job Name
6#MSUB -o Script_Output_CM61-LR-scen-ssp245-r11.000047    # standard output
7#MSUB -e Script_Output_CM61-LR-scen-ssp245-r11.000047    # error output
8#MSUB -eo
9#MSUB -n 940  # Number of MPI tasks (SPMD case) or cores (MPMD case)
10#MSUB -x                    # exclusive node. To specify only for MPMD together with the one below
11#MSUB -T 86400              # Wall clock limit (seconds)
12#MSUB -A scecmip6
13#MSUB -q skylake
14#MSUB -m store,work,scratch
15
16BATCH_NUM_PROC_TOT=$BRIDGE_MSUB_NPROC
17set +x
18
19#**************************************************************
20# Author: Sebastien Denvil
21# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
22# $Revision:: 1460                                     $ Revision of last commit
23# $Author:: jgipsl                                     $ Author of last commit
24# $Date:: 2018-06-25 13:26:49 +0200 (Mon, 25 Jun 2018) $ Date of last commit
25# IPSL (2006)
26#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
27#
28#**************************************************************
29
30#set -eu
31#set -vx
32
33date
34echo
35echo "#######################################"
36echo "#       ANOTHER GREAT SIMULATION      #"
37echo "#######################################"
38echo
39
40#D--------------------------------------------------------------------==
41#D-
42#D-                      Job to launch IGCM models
43#D-
44#D--------------------------------------------------------------------==
45#D-
46
47MODIPSL=/ccc/cont003/dsku/perle1/home/app/gencmip6/devillim/modipsl_v20fev
48libIGCM=${MODIPSL}/libIGCM
49
50#D--------------------------------------------------------------------==
51#D- -1. PLACE FOR USER MODIFICATION
52#D-     - Job Verbosity
53#D-     - PeriodNb
54#D-     - Experience type : DEB(ug), DEV(elopment), RUN
55#D--------------------------------------------------------------------==
56
57#D- Task type DO NOT CHANGE (computing, post-processing or checking)
58TaskType=computing
59
60#D- Increased verbosity (1, 2, 3)
61Verbosity=3
62
63#D- postProcessingStopLevel (0,1,2,3)
64#D- 3 stop the main job if any post-processing went wrong (atlas, monitoring, metrics, create_ts, create_se, rebuild or pack_*)
65#D- 2 stop only if create_ts, create_se or pack_* failed  (dont stop if atlas, monitoring or metrics failed)
66#D- 1 stop only if pack_* failed (dont stop if atlas, monitoring, metrics, create_ts or create_se failed)
67#D- 0 never stop the main job for post-processing that went wrong
68postProcessingStopLevel=1
69
70#D- Experience type : DEB(ug), DEV(elopment), RUN (default)
71#D- DEB : set -vx mode + output files are NOT read only
72#D- DEV : output files are NOT read only
73#D- RUN : output files ARE read only
74JobType=RUN
75
76#D- Number of execution in one job
77PeriodNb=1
78
79#D-
80#D- --------------------------------------------------------------------------------------------------#
81#D-                          ! OPTIONNAL FEATURES /!\ USE WITH CARE !                                 #
82#D- --------------------------------------------------------------------------------------------------#
83
84
85#D- Low level debug : to perform lib test checks, stack construction and IO functions instrumentation
86#D- Default=true
87#D-
88DEBUG_debug=true
89
90#D- Messaging : all activities and call stacks will be sent to ipsl servers
91#D- If true will imply DEBUG_debug=true
92#D- Default=false
93#D-
94BigBrother=true
95
96#D- Set DEBUG_sys to false to disable output calls of function
97#D- true when JobType=DEB # means DEB(ug)
98#D-
99#DEBUG_sys=false
100
101#D- Define running directory
102#D- Default=${TMPDIR} ie temporary batch directory
103#D-
104#RUN_DIR_PATH=/workdir/or/scratchdir/of/this/machine
105
106#D- Define submit directory
107#D- Default= where you launch qsub (variable from scheduler)
108#D-
109#SUBMIT_DIR=$( pwd )
110
111#D- Define input file root directory
112#D- Default=/IGCMG/common/account/of/this/machine
113#D- IPSL:   /prodigfs/ipslfs/igcmg/IGCM
114#D- TGCC:   /ccc/work/cont003/igcmg/igcmg/IGCM
115#D- IDRIS:  /workgpfs/rech/psl/rpsl035/IGCM
116#D- OBELIX: /home/orchideeshare/igcmg/IGCM
117#D- OTHER:  /home/${LOGIN}/IGCM
118
119#D- Change below to override the default
120#R_IN=/u/rech/por/rpor111/DATA
121
122#D- Turn in dry run mode ? (0,1,2,3)
123#D- Default=0
124#D-
125#DRYRUN=3
126#D-# YOU HAVE TO COMPILE YOUR EXE FILES to USE DRYRUN MODE !
127#D-# -------------------------------------------------------------------------------------
128#D-# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
129#D-# |          |  Cp/Exe/param/files |            |         |                           |
130#D-# |          |  Chmod Qsub         |            |         |                           |
131#D-# -------------------------------------------------------------------------------------
132#D-# |    0     |       yes           |    yes     |  yes    |      yes                  |
133#D-# -------------------------------------------------------------------------------------
134#D-# |    1     |       yes           |    yes     |  yes    |      no                   |
135#D-# -------------------------------------------------------------------------------------
136#D-# |    2     |       yes           |    yes     |  no     |      no                   |
137#D-# -------------------------------------------------------------------------------------
138#D-# |    3     |       yes           |    no      |  no     |      no                   |
139#D-# -------------------------------------------------------------------------------------
140
141#D-
142# --------------------------------------------------------------------------------------------------#
143#D- ! END OF OPTIONNAL FEATURES /!\ DO NOT MODIFY ANYTHING BELOW UNLESS YOU KNOW WHAT YOU ARE DOING #
144# --------------------------------------------------------------------------------------------------#
145
146#D-
147#D--------------------------------------------------------------------==
148#D- 0.0 SYSTEM ENVIRONMENT
149#D-     - Define variables for Number of processors
150#D-     - Define MPI variables
151#D-     - Define batch scheduler variables
152#D-     - Source IGCM Library
153#D-     - Get RESOLution in .resol file (temporary)
154#D--------------------------------------------------------------------==
155
156#---------------------------------------------------------------------==
157
158. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
159. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
160. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
161#-------
162. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
163. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
164. ${libIGCM}/libIGCM_comp/libIGCM_comp.ksh
165. ${libIGCM}/libIGCM_post/libIGCM_post.ksh
166#-------
167( ${DEBUG_debug} ) && IGCM_debug_Check
168( ${DEBUG_debug} ) && IGCM_card_Check
169( ${DEBUG_debug} ) && IGCM_date_Check
170
171#--------------------------------------------------------------------==
172
173#-- Resolution FLAG for oasis input
174#RESOL="ORCA2xLMD9671"
175[ -f ${SUBMIT_DIR}/../.resol ] && RESOL=$(head -1 ${SUBMIT_DIR}/../.resol)
176#-- Chemistrie FLAG to ease chemistries configurations management
177#CHEM="AER"
178[ -f ${SUBMIT_DIR}/../.chimie ] && eval $(grep CHEM ${SUBMIT_DIR}/../.chimie)
179
180#D-
181#D--------------------------------------------------------------------==
182#D- 0.1 COMMON ENVIRONMENT
183#D-     - Read libIGCM compatibility version in config.card
184#D-     - Read UserChoices section
185#D-     - Read Ensemble section
186#D-     - Read Post section
187#D-     - Define all netcdf output directories
188#D--------------------------------------------------------------------==
189IGCM_config_CommonConfiguration ${SUBMIT_DIR}/config.card
190
191if [ ! -r ${SUBMIT_DIR}/run.card ] ; then
192  #================================================#
193  #         The file run.card doesn't exist        #
194  #================================================#
195  FirstInitialize=true
196  #copy initial run.card
197  IGCM_sys_Cp ${SUBMIT_DIR}/run.card.init ${SUBMIT_DIR}/run.card
198  IGCM_debug_Print 2 "run.card copied from run.card.init"
199else
200  FirstInitialize=false
201  IGCM_debug_Print 2 "run.card exists"
202fi
203
204# ------------------------------------------------------------------
205# Activate BigBrother so as to supervise this simulation
206# ------------------------------------------------------------------
207IGCM_debug_BigBro_Initialize
208
209#D-
210#D--------------------------------------------------------------------==
211#D- 1. INITIALIZE CONFIGURATION
212#D-    - Simulation configuration
213#D-    - Simulation parameters
214#D-    - Execution parameter
215#D-    - Define input files directory
216#D-    - Read or initialize CumulPeriod
217#D-    - run.card
218#D--------------------------------------------------------------------==
219IGCM_config_Initialize
220
221# --------------------------------------------------------------------==
222# Define, create and cd RUN_DIR
223# --------------------------------------------------------------------==
224RUN_DIR=${RUN_DIR_PATH}/${config_UserChoices_JobName}.${$}
225IGCM_sys_MkdirWork ${RUN_DIR}
226IGCM_sys_Cd ${RUN_DIR}
227
228# ------------------------------------------------------------------
229# Test if all was right before proceeding further
230# ------------------------------------------------------------------
231IGCM_debug_Verif_Exit
232
233#D-
234#D---------------------------------------------------------------------==
235#D- 2. INITIALIZE ALL COMPONENT OF THE CONFIGURATION
236#D-    - Define component executable
237#D-    - Define Write Frequency by component "1M" "1D" ...
238#D-    - Source ${comp}.driver
239#D-    - Define Storage Places by component
240#D-    - ${comp}_Initialize :
241#D-    - Patterns to sed
242#D-    - Variables from namelist
243#D---------------------------------------------------------------------==
244IGCM_comp_Initialize
245
246#D-
247#D---------------------------------------------------------------------==
248#D- 3. PERFORM GENERAL VERIFICATION
249#D-    - Verify compatibility of period length, write frequencies, ...
250#D-    - Verify compatibility of rebuild choices and post-process choices
251#D-    - Will stop here if something is wrong
252#D---------------------------------------------------------------------==
253IGCM_config_Check
254
255# ------------------------------------------------------------------
256# Test if all was right before entering the period loop
257# ------------------------------------------------------------------
258IGCM_debug_Verif_Exit
259
260#D-
261#D---------------------------------------------------------------------==
262#D- 4. ENTER THE DEEP EXECUTION LOOP
263#D-    ! FASTEN YOUR SEAT BELTS PLEASE. !
264#D-    !   WE WILL CROSS TURBULENCES.   !
265#D---------------------------------------------------------------------==
266
267Period=1
268
269PeriodContinue=false
270if [ ${Period} -le ${PeriodNb} ]; then
271  PeriodContinue=true
272fi
273while ( ${PeriodContinue} ) ; do
274
275  echo
276  echo "Starting iteration ${Period} / ${PeriodNb}"
277
278  #D-
279  # ------------------------------------------------------------------
280  #D-   4.1 COMPUTE AND DEFINE DATE INFORMATION.
281  #D-       - ${PeriodDateBegin}
282  #D-       - ${PeriodDateEnd}
283  #D-       - ${CumulPeriod}
284  #D-       - ${DatesPeriod}=${PeriodDateBegin}_${PeriodDateEnd}
285  #D-       - ${PeriodLength} => JOURS=31 , 28, 29, 30 ; MOIS=0 ; ANS=0
286  #D-       - update run.card value
287  # ------------------------------------------------------------------
288  IGCM_config_PeriodStart
289
290  #D-
291  # ------------------------------------------------------------------
292  #D-   4.2 BEGIN COMPUTATION ONLY IF NEEDED
293  # ------------------------------------------------------------------
294  if [ ${SimulationLengthInDays} -gt ${ExperienceLengthInDays} ] ; then
295    IGCM_debug_Print 1 "break because '\${SimulationLengthInDays} -gt \${ExperienceLengthInDays}' : ${SimulationLengthInDays} -gt ${ExperienceLengthInDays}"
296    break ;
297  fi
298
299  #D-
300  # ------------------------------------------------------------------
301  #D-   4.3 CALL OPTIONNAL FUNCTION WITHIN DRIVER
302  #D-       - to set special variables used in lists (Param, Init or Bc).
303  # ------------------------------------------------------------------
304  IGCM_comp_PeriodStart
305
306  #D-
307  # ------------------------------------------------------------------
308  #D-   4.4 GET PARAMETERS TEXT FILES UPDATED BY JOB
309  #D-       - (.def, namelist ...)
310  #D-       - READ AND USE BY GCM AT EACH EXECUTION.
311  # ------------------------------------------------------------------
312  IGCM_comp_GetInputParametersFiles
313
314  #D-
315  # ------------------------------------------------------------------
316  #D-   4.5 GET INITIAL STATE
317  #D-       - (Etat0, carteveg,relief...)
318  #D-       - NECESSARY ONLY IF CumulPeriod= (=> NUMERO) == 1
319  #D-       - AND ???_Restart=NO
320  #D-       - READ AND USE BY GCM FOR ONLY ONE EXECUTION.
321  # ------------------------------------------------------------------
322  IGCM_comp_GetInputInitialStateFiles
323
324  #D-
325  # ------------------------------------------------------------------
326  #D-   4.6 GET BOUNDARIES CONDITIONS
327  #D-       - (SST, WIND[X,Y,Z], LAI ...)
328  #D-       - READ AND USE BY GCM AT EACH EXECUTION.
329  # ------------------------------------------------------------------
330  IGCM_comp_GetInputBoundaryFiles
331
332  #D-
333  # ------------------------------------------------------------------
334  #D-   4.7 GET SmoothFiles CONDITIONS
335  #D-       - (SST, WIND[X,Y,Z], LAI ...)
336  #D-       - READ AND USE BY GCM AT EACH EXECUTION
337  #D-       - Do not change at each ${Period}
338  # ------------------------------------------------------------------
339  IGCM_comp_GetInputSmoothFiles
340
341  #D-
342  # ------------------------------------------------------------------
343  #D-   4.8 GET RESTART FILES
344  #D-       - (restartphy.nc, orca_restart.nc ...)
345  #D-       - READ AND USE BY GCM AT EACH EXECUTION.
346  #D-
347  #D-       - IF A COMPONENT DO NOT RESTART FROM PREVIOULSY COMPUTED RESTART
348  #D-       - ONLY IF CumulPeriod= (=> NUMERO) == 1
349  #D-       - MUST EXECUTE CREATE ETAT0_LIMIT (TYPICALLY LMDZ AND ./create_etat0_limit.e)
350  #D-
351  #D-       - IF CumulPeriod == 1
352  #D-       - AND A COMPONENT RESTART FROM PREVIOULSY COMPUTED RESTART
353  #D-       - MUST BRING THIS RESTART IN ${RUN_DIR_PATH}
354  #D-
355  #D-       - ELSE BRING RESTART FROM ${JobName} IN ${RUN_DIR_PATH}
356  # ------------------------------------------------------------------
357  IGCM_comp_GetInputRestartFiles
358
359  # ------------------------------------------------------------------
360  #D-   4.9 GET BINARY FILES
361  #D-       - USED BY AT EACH EXECUTION.
362  #D-       - CHECK BINARIES HAS BEEN COMPILED WITH THE SAME COMPILER
363  #D-       - RELOAD THE APPROPRIATE ENVIRONMENT THE FIRST TIME IF NEEDED
364  # ------------------------------------------------------------------
365  IGCM_comp_GetInputBinaryFiles
366
367  # ------------------------------------------------------------------
368  # Test if all was right before Update
369  # ------------------------------------------------------------------
370  IGCM_debug_Verif_Exit
371
372  #D-
373  # ------------------------------------------------------------------
374  #D-   4.10 ACTIVATE RUNNING ENVIRONNEMENT VARIABLES
375  #D-       - including module load associated to compiler & tools
376  #D-       - .env_intel17.0.2_curie
377  #D-       - .env_netcdf4.3.3.1_curie
378  # ------------------------------------------------------------------
379  IGCM_sys_activ_variables
380
381  #D-
382  # ------------------------------------------------------------------
383  #D-   4.11 UPDATE ParametersFiles
384  #D-        - (.def, namelist ...)
385  #D-        - with current value of (ORCA_NIT00, ORCA_LRSTAR, RAZ_DATE, ...)
386  # ------------------------------------------------------------------
387  IGCM_comp_Update
388
389  # ------------------------------------------------------------------
390  # Test if all was right before execution
391  # ------------------------------------------------------------------
392  IGCM_debug_Verif_Exit
393
394  #D-
395  #D---------------------------------------------------------------------==
396  #D- 5. EXECUTION
397  #D---------------------------------------------------------------------==
398
399  echo
400  echo "#######################################"
401  echo "#      DIR BEFORE RUN EXECUTION       #"
402  echo "#######################################"
403  echo
404  ls -lrt
405
406  echo "========================================================================"
407  if [ ${DRYRUN} -le 1 ] ; then
408    REAL_DATE_INIT=$( date )
409    echo                                                                                   > ${Exe_Output}
410    echo "#######################################"                                        >> ${Exe_Output}
411    echo "EXECUTION of : ${EXECUTION}"
412    echo "EXECUTION of : ${EXECUTION}"                                                    >> ${Exe_Output}
413    echo                                                                                  >> ${Exe_Output}
414    typeset RET
415    RUN_DATE_BEGIN=$( date '+%Y-%m-%dT%H:%M:%S' )
416    ${EXECUTION}  >> ${Exe_Output} 2>&1
417    RET=$?
418    RUN_DATE_END=$( date '+%Y-%m-%dT%H:%M:%S' )
419    if [ ${RET} -gt 0 ] ; then
420      echo "Return code of executable :" ${RET}
421      IGCM_debug_Exit "EXECUTABLE"
422      IGCM_sys_Mkdir ${SUBMIT_DIR}/Debug
423      IGCM_sys_Cp ${Exe_Output} ${SUBMIT_DIR}/Debug/${PREFIX}_${Exe_Output}_error
424      ExecutionFail=true
425    fi
426    echo                                            >> ${Exe_Output}
427    echo "#######################################"  >> ${Exe_Output}
428    echo "libIGCM RunDateBegin=${RUN_DATE_BEGIN}"   >> ${Exe_Output}
429    echo "libIGCM RunDateEnd=${RUN_DATE_END}"       >> ${Exe_Output}
430    echo                                            >> ${Exe_Output}
431  else
432    echo "EXECUTION of : ${EXECUTION} simulated for DRYRUN = " $DRYRUN
433    echo "EXECUTION of : ${EXECUTION} simulated for DRYRUN = " $DRYRUN > ${Exe_Output}
434    if ( $DEBUG_debug ) ; then
435      echo "FOR EXECUTION DRYRUN mode = " $DRYRUN >> stack
436    fi
437  fi
438  echo "========================================================================"
439
440  echo
441  echo "#######################################"
442  echo "#       DIR AFTER RUN EXECUTION       #"
443  echo "#######################################"
444  echo
445  ls -lrt
446
447  #D-
448  #D---------------------------------------------------------------------==
449  #D- 6. POST EXECUTION OPERATIONS
450  #D---------------------------------------------------------------------==
451
452  #D-
453  # ------------------------------------------------------------------
454  #D-   6.1. DESACTIVATE RUNNING ENVIRONNEMENT VARIABLES
455  # ------------------------------------------------------------------
456  IGCM_sys_desactiv_variables
457
458  #D-
459  # ------------------------------------------------------------------
460  #D-   6.2. SAVE OUTPUTS
461  #D-        - On archive/storage machine
462  #D-        - netcdf outputs, restarts and text files of models
463  # ------------------------------------------------------------------
464  IGCM_comp_Finalize
465
466  # ------------------------------------------------------------------
467  # Test if all was right after Finalize
468  # ------------------------------------------------------------------
469  IGCM_debug_Verif_Exit
470
471  #D-
472  # ------------------------------------------------------------------
473  #D-   6.3. CONFIGURE POST-PROCESSING
474  #D-        - asynchronous post-treatment (Level 0) when necessary (rebuild/pack)
475  #D-        - classic post-treatment      (Level 1) when necessary (TimeSeries/Seasonal)
476  # ------------------------------------------------------------------
477  IGCM_post_Configure
478
479  #D-
480  # ------------------------------------------------------------------
481  #D-   6.4. SUBMIT POST-PROCESSING
482  #D-        - remote rebuild or/and post-treatment process if necessary
483  # ------------------------------------------------------------------
484  IGCM_post_Submit
485
486  #D-
487  # ------------------------------------------------------------------
488  #D-   6.5. FINALYZE CURRENT LOOP AND PREPARE NEXT ONE
489  #D-        - Manage executable size to save only different binary
490  #D-        - Write in run.card user, system and elapse time
491  #D-        - Check that everything went well ! No ? then we stop.
492  #D-        - Determine next computed period
493  # ------------------------------------------------------------------
494  IGCM_config_PeriodEnd
495
496  echo "Ending iteration ${Period}"
497  (( Period = Period + 1 ))
498
499  # End loop if date end is reached
500  if [ ${SimulationLengthInDays} -ge ${ExperienceLengthInDays} ] ; then
501    break ;
502  fi
503
504  PeriodContinue=false
505  if [ ${Period} -le ${PeriodNb} ]; then
506    PeriodContinue=true
507  fi
508done
509
510#D-
511#D---------------------------------------------------------------------==
512#D- 7. SUBMIT NEXT JOB OR SEND MAIL IF SIMULATION IS OVER.
513#D---------------------------------------------------------------------==
514IGCM_config_Finalize
515
516date
Note: See TracBrowser for help on using the repository browser.