Changeset 1220 for trunk

Timestamp:

08/25/15 16:46:24 (9 years ago)

Author:

sdipsl

Message:

• Any post-procesing errors will stop the computing jobs when SpaceName?=PROD. See #260
• Feature was removed some years ago because postprocessing errors were legions. More robust now.

Location:

trunk/libIGCM

Files:

• libIGCM_config/libIGCM_config.ksh (modified) (5 diffs)
• libIGCM_debug/libIGCM_debug.ksh (modified) (1 diff)

trunk/libIGCM/libIGCM_config/libIGCM_config.ksh

@@ -311 +311 @@
     fi
 
-    #Test state of run in run.card
-    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration PeriodState
-
-    if ( [ ${run_Configuration_PeriodState} != "Start" ] && [ ${run_Configuration_PeriodState} != "Running" ] && [ ${run_Configuration_PeriodState} != "OnQueue" ] && [ ${run_Configuration_PeriodState} != "Continue" ] ) ; then
-      echo
-      IGCM_debug_Print 1 "!! Error in run.card with PeriodState : " ${run_Configuration_PeriodState} "!!"
-      IGCM_debug_Print 1 "Check post-processing jobs carefully by running ${libIGCM}/RunChecker.job -p ${SUBMIT_DIR}"
-      IGCM_debug_Print 1 "Then try running ${libIGCM}/clean_month.job to rerun one period"
-      IGCM_debug_Print 1 "Then try running ${libIGCM}/clean_year.job to rerun more."
-      IGCM_debug_Exit
-      IGCM_debug_Verif_Exit
-    fi
+    # Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
+    IGCM_config_StateCheck
+
+    # And EXIT if not OK
+    IGCM_debug_Verif_Exit
   fi
 
@@ -455 +448 @@
 }
 
+#===================================
+function IGCM_config_StateCheck
+{
+  IGCM_debug_PushStack "IGCM_config_StateCheck"
+
+    #Test state of run in run.card
+    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration PeriodState
+
+    if [ ${run_Configuration_PeriodState} = "Fatal" ] ; then
+      echo
+      IGCM_debug_Print 1 "!! Error in run.card with PeriodState : " ${run_Configuration_PeriodState} "!!"
+      IGCM_debug_Print 1 "Check post-processing jobs carefully by running ${libIGCM}/RunChecker.job -p ${SUBMIT_DIR}"
+      IGCM_debug_Print 1 "Then try running ${libIGCM}/clean_month.job to rerun one period"
+      IGCM_debug_Print 1 "Then try running ${libIGCM}/clean_year.job to rerun more."
+      IGCM_debug_Exit
+    fi
+
+  IGCM_debug_PopStack "IGCM_config_StateCheck"
+}
 
 #===================================
@@ -598 +610 @@
     #================================================#
 
-    #Test state of run in run.card
-    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration PeriodState
-    if ( [ ${run_Configuration_PeriodState} != "Running" ] && [ ${run_Configuration_PeriodState} != "OnQueue" ] && [ ${run_Configuration_PeriodState} != "Continue" ] ) ; then
-      echo
-      IGCM_debug_Print 1 "!! Error in run.card with PeriodState : " ${run_Configuration_PeriodState} "!!"
-      IGCM_debug_Print 1 "Try running ${libIGCM}/clean_month.job to fix this"
-      echo
-      IGCM_debug_Exit
-      IGCM_debug_Verif_Exit
-    fi
+    # Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
+    IGCM_config_StateCheck
+    # And EXIT if not OK
+    IGCM_debug_Verif_Exit
 
     #===================================#
@@ -659 +665 @@
     IGCM_debug_Verif_Exit
 
+    # Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
+    IGCM_config_StateCheck
+    # And EXIT if not OK
+    IGCM_debug_Verif_Exit
+
+    # We can say we are "Running" now.
     IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Running"
   fi
@@ -1250 +1262 @@
   echo
 
+  # Test state of run in run.card. Will schedule an exit if another process setted it to "Fatal"
+  IGCM_config_StateCheck
+
+  # And EXIT if not OK
+  IGCM_debug_Verif_Exit
+
   if [ ${SimulationLengthInDays} -ge ${ExperienceLengthInDays} ] ; then
     #==========================#

trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh

@@ -737 +737 @@
       # If SpaceName is PROD then we stop when post_processing failed
       if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
-        echo "                        EXIT THE JOB."
+        echo "                        EXIT THE POST-PROCESSING JOB."
         echo
         IGCM_debug_CallStack
+
+        # Notify the computing job that something wrong happened.
+        IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
 
         # Mail notification?