Changeset 811

Timestamp:

03/05/13 15:39:16 (11 years ago)

Author:

sdipsl

Message:

• see #104 homegenize sys lib so as to ease maintenance.
• all sys lib must have the same functions (some of them being dummy or not) depending on centre environment
• indentation clean-up (from emacs macro)

Location:

trunk/libIGCM/libIGCM_sys

Files:

• libIGCM_sys_ada.ksh (modified) (44 diffs)
• libIGCM_sys_curie.ksh (modified) (57 diffs)
• libIGCM_sys_default.ksh (modified) (67 diffs)
• libIGCM_sys_iitm.ksh (modified) (28 diffs)
• libIGCM_sys_obelix.ksh (modified) (70 diffs)

trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh

@@ -251 +251 @@
     echo "IGCM_sys_RshPost :" $@
   fi
-
-  # keep standard input to keep it for postpone if ulam don't answer
+  # keep standard input (stdin) for the loop onto temporary file
   cat >/tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
 
@@ -627 +626 @@
   IGCM_debug_PushStack "IGCM_sys_Qsub" $@
 
-  typeset ERROR
-  ERROR=0
+  typeset status
 
   if ( $DEBUG_sys ) ; then
@@ -639 +637 @@
       -e "s:\# \@ error *= .*:\# \@ error = ${Script_Output}:"   \
       $1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
-  cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM ; ERROR=$? ; cd - ;
-
-  if [ ${ERROR} -gt 0 ] ; then
-    echo "IGCM_sys_Qsub : erreur $@"
+  cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM ; status=$? ; cd - ;
+
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
     IGCM_debug_Exit "IGCM_sys_Qsub"
   else
@@ -658 +656 @@
   IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
 
-  typeset ERROR
-  ERROR=0
+  typeset status
 
   if ( $DEBUG_sys ) ; then
@@ -672 +669 @@
       ${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM
 
-  cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM ; ERROR=$? ; cd - ;
-  if [ ${ERROR} -gt 0 ] ; then
-    echo "IGCM_sys_QsubPost : erreur $@."
+  cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM ; status=$? ; cd - ;
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
     IGCM_debug_Exit "IGCM_sys_QsubPost"
   else
     IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
   fi
-
   IGCM_debug_PopStack "IGCM_sys_QsubPost"
 }
@@ -800 +796 @@
 #D-#==================================================
 #D-function IGCM_sys_Miror_libIGCM
-#D-* Purpose: Mirror libIGCM PATH and lib to ulam
+#D-* Purpose: Mirror libIGCM PATH and lib to frontend
 #D-* Examples:
 #D-
@@ -818 +814 @@
 
   if [ ${status} -gt 0 ] ; then
-    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
+    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
     cat out_rsync
   fi
@@ -1214 +1210 @@
   fi
   echo $( basename $1 )
-
   IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
 }
@@ -1615 +1610 @@
 # REBUILD OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_rebuild
+#D-* Purpose: rebuild parallel files
+#D-* Examples:
+#D-
 function IGCM_sys_rebuild {
   IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
@@ -1654 +1654 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_rebuild_station
+#D-* Purpose: rebuild parallel files describing station 
+#D-* Examples:
+#D-
 function IGCM_sys_rebuild_station {
   IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
@@ -1748 +1753 @@
 # Activate Running Environnment Variables
 
+#D-#==================================================
+#D-function IGCM_sys_desactiv_variables
+#D-* Purpose: set environement variables prior to execution
+#D-* Examples:
+#D-
 function IGCM_sys_activ_variables {
   IGCM_debug_PushStack "IGCM_sys_activ_variables"
   if ( $DEBUG_sys ) ; then
-    echo "IGCM_sys_activ_variables "
-  fi
+    echo "IGCM_sys_activ_variables"
+  fi
+
+# --------------------------------------------------------------------
+#D- MPI specifications
+# --------------------------------------------------------------------
+
+# --------------------------------------------------------------------
+#D- Other specifications
+# --------------------------------------------------------------------
 
   ulimit -s unlimited
@@ -1767 +1785 @@
 # Desactivate Running Environnment Variables
 
+#D-#==================================================
+#D-function IGCM_sys_desactiv_variables
+#D-* Purpose: unset environement variables after execution
+#D-* Examples:
+#D-
 function IGCM_sys_desactiv_variables {
   IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
   if ( $DEBUG_sys ) ; then
-    echo "IGCM_sys_desactiv_variables "
+    echo "IGCM_sys_desactiv_variables"
   fi
 # --------------------------------------------------------------------
@@ -1786 +1809 @@
 # Build MPI/OMP scripts run file (dummy function)
 
+#D-#==================================================
+#D-function IGCM_sys_build_run_file
+#D-* Purpose: build run file (deprecated)
+#D-* Examples:
+#D-
 function IGCM_sys_build_run_file {
 
-IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
+  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
 
 }
@@ -1794 +1822 @@
 ############################################################
 # Build MPI/OMP scripts
+
+#D-#==================================================
+#D-function IGCM_sys_build_execution_scripts
+#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
+#D-* Examples:
+#D-
 function IGCM_sys_build_execution_scripts
 {
@@ -1873 +1907 @@
   else # Only one executable. launch it.
 
-    #
     for comp in ${config_ListOfComponents[*]} ; do
 
@@ -1893 +1926 @@
 
 ############################################################
-# Check of space available on temporary filesytems
+# Check of space available on temporary filesytems.
+
+#D-#==================================================
+#D-function IGCM_sys_check_quota
+#D-* Purpose: check user quota. Stop the simulation if quota above 90%
+#D-* Examples:
+#D-
 function IGCM_sys_check_quota {
   IGCM_debug_PushStack "IGCM_sys_check_quota"
   if ( $DEBUG_sys ) ; then
-    echo "IGCM_sys_check_quota "
+    echo "IGCM_sys_check_quota"
   fi
   # Limit of quota (in %)
   limit_quota=90
+
   # Check of the volume
   volume_quota=$(quota_u -w | grep 'Quota soft' | awk '{print $5}')
@@ -1925 +1965 @@
 #D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
 #D-
-
 function IGCM_sys_CountJobInQueue {
   IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
@@ -1943 +1982 @@
 # NCO OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_ncap2
+#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncap2 {
   IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
@@ -1973 +2017 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncap2 : ncap2 error"
-      IGCM_debug_Exit "ncap2"
+    echo "IGCM_sys_ncap2 : ncap2 error"
+    IGCM_debug_Exit "ncap2"
   fi
 
@@ -1980 +2024 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncatted
+#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncatted {
   IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
@@ -2010 +2059 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncatted : ncatted error"
-      IGCM_debug_Exit "ncatted"
+    echo "IGCM_sys_ncatted : ncatted error"
+    IGCM_debug_Exit "ncatted"
   fi
 
@@ -2017 +2066 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncbo
+#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncbo {
   IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
@@ -2047 +2101 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncbo : ncbo error"
-      IGCM_debug_Exit "ncbo"
+    echo "IGCM_sys_ncbo : ncbo error"
+    IGCM_debug_Exit "ncbo"
   fi
 
@@ -2054 +2108 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncdif
+#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncdiff {
   IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
@@ -2084 +2143 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncdiff : ncdiff error"
-      IGCM_debug_Exit "ncdiff"
+    echo "IGCM_sys_ncdiff : ncdiff error"
+    IGCM_debug_Exit "ncdiff"
   fi
 
@@ -2091 +2150 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncea
+#D-* Purpose: encapsulate ncea call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncea {
   IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
@@ -2121 +2185 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncea : ncea error"
-      IGCM_debug_Exit "ncea"
+    echo "IGCM_sys_ncea : ncea error"
+    IGCM_debug_Exit "ncea"
   fi
 
@@ -2128 +2192 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncecat
+#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncecat {
   IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
@@ -2158 +2227 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncecat : ncecat error"
-      IGCM_debug_Exit "ncecat"
+    echo "IGCM_sys_ncecat : ncecat error"
+    IGCM_debug_Exit "ncecat"
   fi
 
@@ -2165 +2234 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncflint
+#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncflint {
   IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
@@ -2195 +2269 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncflint : ncflint error"
-      IGCM_debug_Exit "ncflint"
+    echo "IGCM_sys_ncflint : ncflint error"
+    IGCM_debug_Exit "ncflint"
   fi
 
@@ -2202 +2276 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncks
+#D-* Purpose: encapsulate ncks call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncks {
   IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
@@ -2232 +2311 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncks : ncks error"
-      IGCM_debug_Exit "ncks"
+    echo "IGCM_sys_ncks : ncks error"
+    IGCM_debug_Exit "ncks"
   fi
 
@@ -2239 +2318 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncpdq
+#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncpdq {
   IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
@@ -2269 +2353 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncpdq : ncpdq error"
-      IGCM_debug_Exit "ncpdq"
+    echo "IGCM_sys_ncpdq : ncpdq error"
+    IGCM_debug_Exit "ncpdq"
   fi
 
@@ -2276 +2360 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncra
+#D-* Purpose: encapsulate ncra call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncra {
   IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
@@ -2306 +2395 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncra : ncra error"
-      IGCM_debug_Exit "ncra"
+    echo "IGCM_sys_ncra : ncra error"
+    IGCM_debug_Exit "ncra"
   fi
 
@@ -2313 +2402 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncrcat
+#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncrcat {
   IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
@@ -2353 +2447 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncrcat : ncrcat error"
-      #IGCM_debug_Exit "ncrcat"
+    echo "IGCM_sys_ncrcat : ncrcat error"
+    #IGCM_debug_Exit "ncrcat"
   fi
 
@@ -2360 +2454 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncrename
+#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncrename {
   IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
@@ -2390 +2489 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncrename : ncrename error"
-      IGCM_debug_Exit "ncrename"
+    echo "IGCM_sys_ncrename : ncrename error"
+    IGCM_debug_Exit "ncrename"
   fi
 
@@ -2397 +2496 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncwa
+#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncwa {
   IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
@@ -2427 +2531 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncwa : ncwa error"
-      IGCM_debug_Exit "ncwa"
+    echo "IGCM_sys_ncwa : ncwa error"
+    IGCM_debug_Exit "ncwa"
   fi
 
@@ -2437 +2541 @@
 # CDO OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_cdo
+#D-* Purpose: encapsulate cdo call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_cdo {
   IGCM_debug_PushStack "IGCM_sys_cdo" -- $@

trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh

@@ -248 +248 @@
     ${@}
 EOF
-  if [ $? -gt 0 ] ; then
-    echo "IGCM_sys_RshArchive : erreur."
+  status=$?
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
     IGCM_debug_Exit "IGCM_sys_RshArchive"
   fi
@@ -265 +266 @@
     echo "IGCM_sys_RshPost :" $@
   fi
-
-  #echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
-  #cat tmp_IGCM_sys_RshPost_$$
   # keep standard input (stdin) for the loop onto temporary file
-  cat >tmp_IGCM_sys_RshPost_$$
-
-# ============ FRONTEND START ============ #
-
-  /bin/ksh <tmp_IGCM_sys_RshPost_$$
+  cat >/tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
+
+  /bin/ksh </tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
   if [ $? -gt 0 ] ; then
     echo "IGCM_sys_RshPost : erreur."
@@ -279 +275 @@
   fi
   # delete temporary file
-  \rm tmp_IGCM_sys_RshPost_$$
+  \rm /tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
 
 # ============ FRONTEND  END  ============ #
@@ -372 +368 @@
     mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
   fi
+
   sleep 10
   rm -f job_end.mail
-  
+
   if [ $? -gt 0 ] ; then
     echo "IGCM_sys_SendMail : erreur."
@@ -420 +417 @@
   if [ ! -d ${1} ]; then
     \mkdir -p $1
-    if [ $? -gt 0 ] ; then
-      echo "IGCM_sys_MkdirArchive : erreur."
+    status=$?
+
+    if [ ${status} -gt 0 ] ; then
+      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
       IGCM_debug_Exit "IGCM_sys_MkdirArchive"
     fi
@@ -533 +532 @@
   ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
   IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
-
   return ${ExistFlag}
 }
@@ -672 +670 @@
     echo "IGCM_sys_Qsub :" $@
   fi
-  typeset options
+  typeset options status
   if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
     options=" -o ${Script_Output} -e ${Script_Output}"
@@ -680 +678 @@
 
   /usr/bin/ccc_msub ${options} < $1
-  if [ $? -gt 0 ] ; then
-    echo "IGCM_sys_Qsub : erreur ${options} $1"
+  status=$?
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
     IGCM_debug_Exit "IGCM_sys_Qsub"
   fi
@@ -697 +696 @@
     echo "IGCM_sys_QsubPost :" $@
   fi
+  typeset status
   MY_PROJECT=$(echo ${BRIDGE_MSUB_PROJECT} | cut -d@ -f1 )
   /usr/bin/ccc_msub -Q normal -A ${MY_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job
-  if [ $? -gt 0 ] ; then
-    echo "IGCM_sys_QsubPost : erreur " $@
+  status=$?
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
     IGCM_debug_Exit "IGCM_sys_QsubPost"
   fi
@@ -823 +824 @@
 #D-#==================================================
 #D-function IGCM_sys_Miror_libIGCM
-#D-* Purpose: Mirror libIGCM PATH and lib to cesium
+#D-* Purpose: Mirror libIGCM PATH and lib to frontend
 #D-* Examples:
 #D-
@@ -841 +842 @@
 
   if [ ${status} -gt 0 ] ; then
-    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
+    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
     cat out_rsync
   fi
@@ -920 +921 @@
 
   if [ ${status} -gt 0 ] ; then
-    echo "IGCM_sys_RmRunDir : error code ${status}"
+    echo "IGCM_sys_RmRunDir : rm error code is ${status}."
     cat out_rsync
     IGCM_debug_Exit "IGCM_sys_RmRunDir"
@@ -989 +990 @@
 
     if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_Put_Dir : error code ${status}"
+      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_Put_Dir"
@@ -1035 +1036 @@
       \cp -ur $1 $2 >> out_rsync 2>&1
       status=$?
-      if [ ${status} -gt 0 ]; then
+      if [ ${status} -gt 0 ] ; then
         IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
         IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
@@ -1389 +1390 @@
       if [ "X$( diff $1 $2 )" = X ] ; then
         IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
+        status=0
         skip=true
       else
@@ -1835 +1837 @@
 # REBUILD OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_rebuild
+#D-* Purpose: rebuild parallel files
+#D-* Examples:
+#D-
 function IGCM_sys_rebuild {
   IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
@@ -1874 +1881 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_rebuild_station
+#D-* Purpose: rebuild parallel files describing station 
+#D-* Examples:
+#D-
 function IGCM_sys_rebuild_station {
   IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
@@ -1968 +1980 @@
 # Activate Running Environnment Variables
 
+#D-#==================================================
+#D-function IGCM_sys_desactiv_variables
+#D-* Purpose: set environement variables prior to execution
+#D-* Examples:
+#D-
 function IGCM_sys_activ_variables {
   IGCM_debug_PushStack "IGCM_sys_activ_variables"
@@ -1974 +1991 @@
   fi
 
+# --------------------------------------------------------------------
+#D- MPI specifications
+# --------------------------------------------------------------------
+
+# --------------------------------------------------------------------
+#D- Other specifications
+# --------------------------------------------------------------------
+
   ulimit -s unlimited
 
@@ -1982 +2007 @@
 # Desactivate Running Environnment Variables
 
+#D-#==================================================
+#D-function IGCM_sys_desactiv_variables
+#D-* Purpose: unset environement variables after execution
+#D-* Examples:
+#D-
 function IGCM_sys_desactiv_variables {
   IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
@@ -1987 +2017 @@
     echo "IGCM_sys_desactiv_variables"
   fi
+# --------------------------------------------------------------------
+#D- MPI specifications
+# --------------------------------------------------------------------
+
+# --------------------------------------------------------------------
+#D- Other specifications
+# --------------------------------------------------------------------
+
   IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
 }
@@ -1993 +2031 @@
 # Build MPI/OMP scripts run file (dummy function)
 
+#D-#==================================================
+#D-function IGCM_sys_build_run_file
+#D-* Purpose: build run file (deprecated)
+#D-* Examples:
+#D-
 function IGCM_sys_build_run_file {
 
-IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
+  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
 
 }
@@ -2001 +2044 @@
 ############################################################
 # Build MPI/OMP scripts
+
+#D-#==================================================
+#D-function IGCM_sys_build_execution_scripts
+#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
+#D-* Examples:
+#D-
 function IGCM_sys_build_execution_scripts
 {
@@ -2008 +2057 @@
   fi
 
-    # Verification with MSUB parameter
   EXECUTION=${HOST_MPIRUN_COMMAND}
 
@@ -2022 +2070 @@
 # Then first loop on the components for the coupler ie oasis
 
-### the coupler ie oasis must be the first one
+## the coupler ie oasis must be the first one
     for comp in ${config_ListOfComponents[*]} ; do
 
@@ -2063 +2111 @@
   else # Only one executable. launch it.
 
-    #
     for comp in ${config_ListOfComponents[*]} ; do
 
@@ -2114 +2161 @@
 
 ############################################################
-# Check of space available on temporary filesytems
+# Check of space available on temporary filesytems.
+
+#D-#==================================================
+#D-function IGCM_sys_check_quota
+#D-* Purpose: check user quota. Stop the simulation if quota above 90%
+#D-* Examples:
+#D-
 function IGCM_sys_check_quota {
   IGCM_debug_PushStack "IGCM_sys_check_quota"
@@ -2133 +2186 @@
 
     if [ "${unit_quota}" = "*" ] ; then
-        IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
-        IGCM_debug_Print 1 "More than 100% of your quota is used"
-        IGCM_debug_Print 1 "Use the ccc_quota command to check"
-        IGCM_debug_Print 1 "You must have more than 10% available to run"
-        IGCM_debug_Exit "Not enough space to run ! STOP HERE"
-        IGCM_debug_Verif_Exit
+      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
+      IGCM_debug_Print 1 "More than 100% of your quota is used"
+      IGCM_debug_Print 1 "Use the ccc_quota command to check"
+      IGCM_debug_Print 1 "You must have more than 10% available to run"
+      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
+      IGCM_debug_Verif_Exit
     fi
 
@@ -2196 +2249 @@
 
     if [ "${unit_quota}" = "*" ] ; then
-        IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
-        IGCM_debug_Print 1 "More than 100% of your quota is used"
-        IGCM_debug_Print 1 "Use the ccc_quota command to check"
-        IGCM_debug_Print 1 "You must have more than 10% available to run"
-        IGCM_debug_Exit "Not enough space to run ! STOP HERE"
-        IGCM_debug_Verif_Exit
+      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
+      IGCM_debug_Print 1 "More than 100% of your quota is used"
+      IGCM_debug_Print 1 "Use the ccc_quota command to check"
+      IGCM_debug_Print 1 "You must have more than 10% available to run"
+      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
+      IGCM_debug_Verif_Exit
     fi
 
@@ -2255 +2308 @@
 #D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
 #D-
-
 function IGCM_sys_CountJobInQueue {
   IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
@@ -2275 +2327 @@
 # NCO OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_ncap2
+#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncap2 {
   IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
@@ -2305 +2362 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncap2 : ncap2 error"
-      IGCM_debug_Exit "ncap2"
+    echo "IGCM_sys_ncap2 : ncap2 error"
+    IGCM_debug_Exit "ncap2"
   fi
 
@@ -2312 +2369 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncatted
+#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncatted {
   IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
@@ -2342 +2404 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncatted : ncatted error"
-      IGCM_debug_Exit "ncatted"
+    echo "IGCM_sys_ncatted : ncatted error"
+    IGCM_debug_Exit "ncatted"
   fi
 
@@ -2349 +2411 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncbo
+#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncbo {
   IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
@@ -2379 +2446 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncbo : ncbo error"
-      IGCM_debug_Exit "ncbo"
+    echo "IGCM_sys_ncbo : ncbo error"
+    IGCM_debug_Exit "ncbo"
   fi
 
@@ -2386 +2453 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncdif
+#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncdiff {
   IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
@@ -2416 +2488 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncdiff : ncdiff error"
-      IGCM_debug_Exit "ncdiff"
+    echo "IGCM_sys_ncdiff : ncdiff error"
+    IGCM_debug_Exit "ncdiff"
   fi
 
@@ -2423 +2495 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncea
+#D-* Purpose: encapsulate ncea call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncea {
   IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
@@ -2453 +2530 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncea : ncea error"
-      IGCM_debug_Exit "ncea"
+    echo "IGCM_sys_ncea : ncea error"
+    IGCM_debug_Exit "ncea"
   fi
 
@@ -2460 +2537 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncecat
+#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncecat {
   IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
@@ -2490 +2572 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncecat : ncecat error"
-      IGCM_debug_Exit "ncecat"
+    echo "IGCM_sys_ncecat : ncecat error"
+    IGCM_debug_Exit "ncecat"
   fi
 
@@ -2497 +2579 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncflint
+#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncflint {
   IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
@@ -2527 +2614 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncflint : ncflint error"
-      IGCM_debug_Exit "ncflint"
+    echo "IGCM_sys_ncflint : ncflint error"
+    IGCM_debug_Exit "ncflint"
   fi
 
@@ -2534 +2621 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncks
+#D-* Purpose: encapsulate ncks call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncks {
   IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
@@ -2564 +2656 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncks : ncks error"
-      IGCM_debug_Exit "ncks"
+    echo "IGCM_sys_ncks : ncks error"
+    IGCM_debug_Exit "ncks"
   fi
 
@@ -2571 +2663 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncpdq
+#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncpdq {
   IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
@@ -2601 +2698 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncpdq : ncpdq error"
-      IGCM_debug_Exit "ncpdq"
+    echo "IGCM_sys_ncpdq : ncpdq error"
+    IGCM_debug_Exit "ncpdq"
   fi
 
@@ -2608 +2705 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncra
+#D-* Purpose: encapsulate ncra call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncra {
   IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
@@ -2638 +2740 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncra : ncra error"
-      IGCM_debug_Exit "ncra"
+    echo "IGCM_sys_ncra : ncra error"
+    IGCM_debug_Exit "ncra"
   fi
 
@@ -2645 +2747 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncrcat
+#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncrcat {
   IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
@@ -2685 +2792 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncrcat : ncrcat error"
-      #IGCM_debug_Exit "ncrcat"
+    echo "IGCM_sys_ncrcat : ncrcat error"
+    #IGCM_debug_Exit "ncrcat"
   fi
 
@@ -2692 +2799 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncrename
+#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncrename {
   IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
@@ -2722 +2834 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncrename : ncrename error"
-      IGCM_debug_Exit "ncrename"
+    echo "IGCM_sys_ncrename : ncrename error"
+    IGCM_debug_Exit "ncrename"
   fi
 
@@ -2729 +2841 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncwa
+#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncwa {
   IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
@@ -2759 +2876 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncwa : ncwa error"
-      IGCM_debug_Exit "ncwa"
+    echo "IGCM_sys_ncwa : ncwa error"
+    IGCM_debug_Exit "ncwa"
   fi
 
@@ -2769 +2886 @@
 # CDO OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_cdo
+#D-* Purpose: encapsulate cdo call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_cdo {
   IGCM_debug_PushStack "IGCM_sys_cdo" -- $@

trunk/libIGCM/libIGCM_sys/libIGCM_sys_default.ksh

@@ -115 +115 @@
 
 #====================================================
-#- ARCHIVE (dedicated to large files)
-typeset -r ARCHIVE=/home/${LOGIN}
+#- ARCHIVE
+typeset ARCHIVE=${ARCHIVE:=/home/${LOGIN}}
 
 #====================================================
 #- STORAGE (dedicated to small/medium files)
-typeset -r STORAGE=${ARCHIVE}
+typeset STORAGE=${ARCHIVE}
 
 #====================================================
@@ -165 +165 @@
 #D-
 function IGCM_sys_ChangeArchive {
-
   IGCM_debug_Print 1 " dummy function : IGCM_sys_ChangeArchive "
-
 }
 
@@ -201 +199 @@
     ${@}
 EOF
-  if [ $? -gt 0 ] ; then
-    echo "IGCM_sys_RshArchive : erreur."
+  status=$?
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
     IGCM_debug_Exit "IGCM_sys_RshArchive"
   fi
@@ -219 +218 @@
   fi
 
-  #echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
-  #cat tmp_IGCM_sys_RshPost_$$
   # keep standard input (stdin) for the loop onto temporary file
   cat >tmp_IGCM_sys_RshPost_$$
@@ -231 +228 @@
   # delete temporary file
   \rm tmp_IGCM_sys_RshPost_$$
-
   IGCM_debug_PopStack "IGCM_sys_RshPost"
 }
@@ -270 +266 @@
   fi
 
+  sleep 10
+  rm -f job_end.mail
+
   if [ $? -gt 0 ] ; then
     echo "IGCM_sys_SendMail : erreur."
@@ -313 +312 @@
   fi
   #- creation de repertoire sur le serveur fichier
-  if [ ! -d ${1} ]; then
-    \mkdir -p $1
-    if [ $? -gt 0 ] ; then
-      echo "IGCM_sys_MkdirArchive : erreur."
-      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
-    fi
+  mkdir -p $1
+  status=$?
+
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
+    IGCM_debug_Exit "IGCM_sys_MkdirArchive"
   fi
   IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
@@ -428 +427 @@
   ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
   IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
-
   return ${ExistFlag}
 }
@@ -712 +710 @@
 
 #D-#==================================================
+#D-function IGCM_sys_Miror_libIGCM
+#D-* Purpose: Mirror libIGCM PATH and lib to frontend
+#D-* Examples:
+#D-
+function IGCM_sys_Mirror_libIGCM {
+  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
+  if ( $DEBUG_sys ) ; then
+    echo "IGCM_sys_Mirror_libIGCM"
+  fi
+
+  typeset status
+
+  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
+
+  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
+  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
+  status=$?
+
+  if [ ${status} -gt 0 ] ; then
+    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
+    cat out_rsync
+  fi
+  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
+}
+
+#D-#==================================================
 #D-function IGCM_sys_Cp
 #D-* Purpose: generic cp
@@ -841 +865 @@
 
     if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_Put_Dir : error code ${status}"
+      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_Put_Dir"
@@ -870 +894 @@
     fi
 
-    typeset RET
+    typeset status
 
     #USUAL WAY
-    \cp -R $1 $2 > out_rsync 2>&1
-    RET=$?
-
-    if [ ${RET} -gt 0 ] ; then
-      echo "IGCM_sys_Get_Dir : error."
+    \cp -ur $1 $2 > out_rsync 2>&1
+    status=$?
+
+    if [ ${status} -gt 0 ] ; then
+      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_Get_Dir"
@@ -906 +930 @@
     fi
 
-    typeset RET
-
-    #USUAL WAY
-    cp -RL $1 $2 > out_rsync 2>&1
-    RET=$?
-
-    if [ ${RET} -gt 0 ] ; then
+    typeset NB_ESSAI DELAI status i
+    # number of tentative
+    NB_ESSAI=3
+    # time delay between tentative
+    DELAI=2
+
+    i=0
+    while [ $i -lt $NB_ESSAI ] ; do
+      \cp -urL $1 $2 > out_rsync 2>&1
+      status=$?
+      if [ ${status} -gt 0 ]; then
+        IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
+        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
+        sleep $DELAI
+      else
+        break
+      fi
+      (( i = i + 1 ))
+    done
+
+    if [ ${status} -gt 0 ] ; then
       echo "IGCM_sys_Get_Master : error."
       cat out_rsync
@@ -924 +962 @@
   IGCM_debug_PopStack "IGCM_sys_Get_Master"
 }
+
+#====================================================
+#- Call IGCM_sys_Mirror_libIGCM now !
+if ( $MirrorlibIGCM ) ; then
+  IGCM_sys_Mirror_libIGCM
+fi
 
 #D-#==================================================
@@ -947 +991 @@
       IGCM_sys_Chmod 444 ${1}
     fi
+    #
+    # Only if we use rsync
+    #IGCM_sys_MkdirArchive $( dirname $2 )
+    #
+    #USUAL WAY
+    \cp $1 $2 > out_rsync 2>&1
+    status=$?
+
+#       #RSYNC WITH NETWORK SSH CALL
+#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
+#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
+
+#       #RSYNC WITH NFS USE
+#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
+#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
+
+#       status=$?
+#       IGCM_sys_Rsync_out $status
+
+#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
+#       (( status=status+$? ))
+
+    if [ ${status} -gt 0 ] ; then
+      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
+      cat out_rsync
+      IGCM_debug_Exit "IGCM_sys_Put_Rest"
+    else
+      \rm out_rsync
+    fi
+  else
+    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
+  fi
+  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
+}
+
+#D-#==================================================
+#D-function IGCM_sys_PutBuffer_Rest
+#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
+#D-           File and target directory must exist.
+#D-* Examples:
+#D-
+function IGCM_sys_PutBuffer_Rest {
+  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
+  if ( $DEBUG_sys ) ; then
+    echo "IGCM_sys_PutBuffer_Rest :" $@
+  fi
+  if [ $DRYRUN = 0 ]; then
+    if [ ! -f ${1} ] ; then
+      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
+      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
+    fi
+
+    typeset status
+    #
+    if [ X${JobType} = XRUN ] ; then
+      IGCM_sys_Chmod 444 ${1}
+    fi
 
     #
@@ -953 +1054 @@
     status=$?
 
-#       #RSYNC WITH NETWORK SSH CALL
-#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
-#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
-
-#       #RSYNC WITH NFS USE
-#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
-#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
-
-#       status=$?
-#       IGCM_sys_Rsync_out $status
-
-#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
-#       (( status=status+$? ))
-
-    if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
-      cat out_rsync
-      IGCM_debug_Exit "IGCM_sys_Put_Rest"
-    else
-      \rm out_rsync
-    fi
-  else
-    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
-  fi
-  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
-}
-
-#D-#==================================================
-#D-function IGCM_sys_PutBuffer_Rest
-#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
-#D-           File and target directory must exist.
-#D-* Examples:
-#D-
-function IGCM_sys_PutBuffer_Rest {
-  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
-  if ( $DEBUG_sys ) ; then
-    echo "IGCM_sys_PutBuffer_Rest :" $@
-  fi
-  if [ $DRYRUN = 0 ]; then
-    if [ ! -f ${1} ] ; then
-      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
-      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
-    fi
-
-    typeset status
-    #
-    if [ X${JobType} = XRUN ] ; then
-      IGCM_sys_Chmod 444 ${1}
-    fi
-
-    #
-    # USUAL WAY
-    \cp $1 $2 > out_rsync 2>&1
-    status=$?
-
     if [ ${status} -gt 0 ] ; then
       echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
+      [ -f ${2} ] && ls -l ${2}
+      [ -f ${2}/${1} ] && ls -l ${2}/${1}
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
@@ -1042 +1090 @@
     echo "IGCM_sys_Put_Out :" $@
   fi
+
+  typeset status
+
   if [ $DRYRUN = 0 ]; then
-    if [ -f ${1} ] ; then
-      if [ ! -d $( dirname $2 ) ] ; then
-        IGCM_sys_MkdirArchive $( dirname $2 )
-      fi
-    else
+    if [ ! -f ${1} ] ; then
       echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
       IGCM_debug_PopStack "IGCM_sys_Put_Out"
       return 1
     fi
-
-    typeset RET
+    #
+    IGCM_sys_MkdirArchive $( dirname $2 )
     #
     if [ X${JobType} = XRUN ] ; then
@@ -1064 +1111 @@
     echo ${RSYNC} ${RSYNC_opt} $1 $2 > out_rsync 2>&1
     ${RSYNC} ${RSYNC_opt} $1 $2 >> out_rsync 2>&1
-    RET=$?
-    IGCM_sys_Rsync_out $RET
+    status=$?
+    IGCM_sys_Rsync_out $status
 
     ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
-    (( RET=RET+$? ))
-
-    if [ ${RET} -gt 0 ] ; then
-      echo "IGCM_sys_Put_Out : error."
+    (( status=status+$? ))
+
+    if [ ${status} -gt 0 ] ; then
+      IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_Put_Out"
@@ -1094 +1141 @@
     echo "IGCM_sys_PutBuffer_Out :" $@
   fi
+
+  typeset NB_ESSAI DELAI status i exist skip
+
+  # number of tentative
+  NB_ESSAI=3
+  # time delay between tentative
+  DELAI=2
+
   if [ $DRYRUN = 0 ]; then
     if [ ! -f ${1} ] ; then
@@ -1103 +1158 @@
     IGCM_sys_Mkdir $( dirname $2 )
     #
-    typeset RET
-
-    if [ X${JobType} = XRUN ] ; then
-      if [ X${3} = X ] ; then
-        IGCM_sys_Chmod 444 ${1}
+
+    exist=false
+    skip=false
+    if [ -f $2 ] ; then
+      IGCM_debug_Print 1 "$2 already exist"
+      exist=true
+      if [ "X$( diff $1 $2 )" = X ] ; then
+        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
+        status=0
+        skip=true
+      else
+        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
+        skip=false
       fi
     fi
     #
-    # USUAL WAY
-    \cp $1 $2 > out_rsync 2>&1
-    RET=$?
-
-    if [ ${RET} -gt 0 ] ; then
+    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
+      IGCM_sys_Chmod u+w $2
+    fi
+
+    if [ X${skip} = Xfalse ] ; then
+      i=0
+      while [ $i -lt $NB_ESSAI ] ; do
+        # USUAL WAY
+        \cp $1 $2 > out_rsync 2>&1
+        status=$?
+        if [ ${status} -gt 0 ]; then
+          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
+          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
+          [ -f ${2} ] && ls -l ${2}
+          [ -f ${2}/${1} ] && ls -l ${2}/${1}
+          sleep $DELAI
+        else
+          break
+        fi
+        (( i = i + 1 ))
+      done
+    fi
+
+    if [ ${status} -gt 0 ] ; then
       echo "IGCM_sys_PutBuffer_Out : error."
+      [ -f ${2} ] && ls -l ${2}
+      [ -f ${2}/${1} ] && ls -l ${2}/${1}
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
     else
+
+      if [ X${JobType} = XRUN ] ; then
+        if [ X${3} = X ] ; then
+          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
+          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
+        fi
+      fi
+
       \rm out_rsync
     fi
@@ -1137 +1229 @@
   IGCM_debug_PushStack "IGCM_sys_Get" $@
 
-  typeset DEST RET dm_liste ifile target
+  typeset DEST status dm_liste
 
   if ( $DEBUG_sys ) ; then
@@ -1153 +1245 @@
     # test if the (first) file is present in the old computation :
     IGCM_sys_TestFileArchive ${dm_liste[0]}
-    RET=$?
-    if [ ${RET} -gt 0 ] ; then
+    status=$?
+    if [ ${status} -gt 0 ] ; then
       echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
       IGCM_debug_Exit "IGCM_sys_Get"
-            #return
-    fi
-
-    (( RET=0 ))
-    for target in ${dm_liste[*]} ; do
-      \cp ${target} ${DEST} >> out_rsync 2>&1
-      (( RET=RET+$? ))
-    done
-
-#       echo ${RSYNC} ${RSYNC_opt} $@ > out_rsync 2>&1
-#       ${RSYNC} ${RSYNC_opt} $@ >> out_rsync 2>&1
-#       RET=$?
-#       IGCM_sys_Rsync_out $RET
-
-#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
-#       (( RET=RET+$? ))
-
-    if [ ${RET} -gt 0 ] ; then
-      echo "IGCM_sys_Get : error"
+      return
+    fi
+
+    #USUAL WAY
+    \cp ${dm_liste[*]} ${DEST} > out_rsync 2>&1
+    status=$?
+
+    if [ ${status} -gt 0 ] ; then
+      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
       cat out_rsync
-      \rm out_rsync
       IGCM_debug_Exit "IGCM_sys_Get"
     else
@@ -1196 +1277 @@
   IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
 
-  typeset DEST RET status buf_liste ifile target file_work
+  typeset DEST buf_liste target file_work
+  typeset NB_ESSAI DELAI status i
 
   if ( $DEBUG_sys ) ; then
     echo "IGCM_sys_GetBuffer :" $@
   fi
+
+  # number of tentative
+  NB_ESSAI=3
+  # time delay between tentative
+  DELAI=2
+
   if [ $DRYRUN -le 2 ]; then
     if [ X${1} = X'/l' ] ; then
@@ -1212 +1300 @@
     #USUAL WAY
     if [ X${1} = X'/l' ] ; then
-      (( RET=0 ))
       for target in ${buf_liste[*]} ; do
         local_file=$( basename ${target} )
-        \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
-        (( RET = RET + $? ))
+        i=0
+        while [ $i -lt $NB_ESSAI ] ; do
+          \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
+          status=$?
+          if [ ${status} -gt 0 ]; then
+            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
+            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
+            sleep $DELAI
+          else
+            break
+          fi
+          (( i = i + 1 ))
+        done
+        if [ ${status} -gt 0 ] ; then
+          echo "IGCM_sys_Get : error"
+          cat out_rsync
+          \rm out_rsync
+          IGCM_debug_Exit "IGCM_sys_GetBuffer"
+        else
+          \rm out_rsync
+        fi
       done
     else
-      \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
-      status=$?
-    fi
-
-    if [ ${RET} -gt 0 ] ; then
-      echo "IGCM_sys_GetBuffer : error"
-      cat out_rsync
-      \rm out_rsync
-      IGCM_debug_Exit "IGCM_sys_GetBuffer"
-    else
-      \rm out_rsync
+      i=0
+      while [ $i -lt $NB_ESSAI ] ; do
+        \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
+        status=$?
+        if [ ${status} -gt 0 ]; then
+          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
+          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
+          sleep $DELAI
+        else
+          break
+        fi
+        (( i = i + 1 ))
+      done
+      if [ ${status} -gt 0 ] ; then
+        echo "IGCM_sys_Get : error"
+        cat out_rsync
+        \rm out_rsync
+        IGCM_debug_Exit "IGCM_sys_GetBuffer"
+      else
+        \rm out_rsync
+      fi
     fi
   else
@@ -1286 +1402 @@
 #D-#==================================================
 #D-function IGCM_sys_Dods_Cp
-#D-* Purpose: Copy from $(ARCHIVE) files to /tmp/DODS for simulation of internet protocole.
+#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
 #D-* Examples:
 #D-
@@ -1298 +1414 @@
 #D-#==================================================
 #D-function IGCM_sys_Put_Dods
-#D-* Purpose: Put $(ARCHIVE) files on /tmp/DODS for simulation of internet protocole.
+#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
 #D-* Examples:
 #D-
@@ -1306 +1422 @@
     echo "IGCM_sys_Put_Dods :" $@
   fi
-  if [ $DRYRUN = 0 ]; then
-    RET=0
-
-    if [ ${RET} -gt 0 ] ; then
-      echo "IGCM_sys_Put_Dods : error."
-      IGCM_debug_Exit "IGCM_sys_Put_Dods"
-    fi
-  else
-    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
-  fi
   IGCM_debug_PopStack "IGCM_sys_Put_Dods"
 }
@@ -1322 +1428 @@
 # REBUILD OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_rebuild
+#D-* Purpose: rebuild parallel files
+#D-* Examples:
+#D-
 function IGCM_sys_rebuild {
   IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
@@ -1327 +1438 @@
     echo "IGCM_sys_rebuild :" $@
   fi
-  rebuild -f -o $@
-  if [ $? -gt 0 ] ; then
-    echo "IGCM_sys_rebuild : rebuild error"
+
+  typeset NB_ESSAI DELAI status i firstArg
+  # number of tentative
+  NB_ESSAI=3
+  # time delay between tentative
+  DELAI=2
+
+  i=0
+  while [ $i -lt $NB_ESSAI ] ; do
+    rebuild -f -o $@ > out_rsync 2>&1
+    status=$?
+    if [ ${status} -gt 0 ] ; then
+      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
+      cat out_rsync
+      \rm out_rsync
+      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
+      firstArg=${1}
+      \rm ${firstArg}
+      sleep $DELAI
+    else
+      \rm out_rsync
+      break
+    fi
+    (( i = i + 1 ))
+  done
+
+  if [ ${status} -gt 0 ] ; then
+    echo "IGCM_sys_rebuild : rebuild error code is ${status}"
     IGCM_debug_Exit "rebuild"
   fi
@@ -1336 +1472 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_rebuild_station
+#D-* Purpose: rebuild parallel files describing station 
+#D-* Examples:
+#D-
 function IGCM_sys_rebuild_station {
   IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
@@ -1430 +1571 @@
 # Activate Running Environnment Variables
 
+#D-#==================================================
+#D-function IGCM_sys_desactiv_variables
+#D-* Purpose: set environement variables prior to execution
+#D-* Examples:
+#D-
 function IGCM_sys_activ_variables {
   IGCM_debug_PushStack "IGCM_sys_activ_variables"
@@ -1435 +1581 @@
     echo "IGCM_sys_activ_variables"
   fi
+
+# --------------------------------------------------------------------
+#D- MPI specifications
+# --------------------------------------------------------------------
+
+# --------------------------------------------------------------------
+#D- Other specifications
+# --------------------------------------------------------------------
+
   IGCM_debug_PopStack "IGCM_sys_activ_variables"
 }
@@ -1441 +1596 @@
 # Desactivate Running Environnment Variables
 
+#D-#==================================================
+#D-function IGCM_sys_desactiv_variables
+#D-* Purpose: unset environement variables after execution
+#D-* Examples:
+#D-
 function IGCM_sys_desactiv_variables {
   IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
@@ -1446 +1606 @@
     echo "IGCM_sys_desactiv_variables"
   fi
+# --------------------------------------------------------------------
+#D- MPI specifications
+# --------------------------------------------------------------------
+
+# --------------------------------------------------------------------
+#D- Other specifications
+# --------------------------------------------------------------------
+
   IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
 }
@@ -1452 +1620 @@
 # Build MPI/OMP scripts run file (dummy function)
 
+#D-#==================================================
+#D-function IGCM_sys_build_run_file
+#D-* Purpose: build run file (deprecated)
+#D-* Examples:
+#D-
 function IGCM_sys_build_run_file {
 
@@ -1460 +1633 @@
 ############################################################
 # Build MPI/OMP scripts
+
+#D-#==================================================
+#D-function IGCM_sys_build_execution_scripts
+#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
+#D-* Examples:
+#D-
 function IGCM_sys_build_execution_scripts
 {
@@ -1471 +1650 @@
 
   if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
-    IGCM_debug_Exit "IGCM_sys_titane build_execution_scripts : Job_${config_UserChoices_JobName} doesn't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
+    IGCM_debug_Exit "IGCM_sys_default build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
   fi
 
@@ -1712 +1891 @@
   fi
 
-  IGCM_debug_Print 1 "sys $( hostname ) : La commande d execution est "
-  IGCM_debug_Print 1 $EXECUTION
+  IGCM_debug_Print 1 "sys $( hostname ) : execution command is"
+  IGCM_debug_Print 1 "$EXECUTION"
 
   IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
@@ -1719 +1898 @@
 
 ############################################################
-# Check of space available on temporary filesytems
+# Check of space available on temporary filesytems. Dummy function here
+
+#D-#==================================================
+#D-function IGCM_sys_check_quota. Dummy call here
+#D-* Purpose: check user quota. Stop the simulation if quota above 90%
+#D-* Examples:
+#D-
 function IGCM_sys_check_quota {
   IGCM_debug_PushStack "IGCM_sys_check_quota"
@@ -1728 +1913 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_CountJobInQueue
+#D-* Purpose: Check if job_name is currently
+#D-  running or in queue
+#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
+#D-
+function IGCM_sys_CountJobInQueue {
+  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
+  if ( $DEBUG_sys ) ; then
+    echo "IGCM_sys_CountJobInQueue"
+  fi
+  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
+}
+
 ##############################################################
 # NCO OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_ncap2
+#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncap2 {
   IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
@@ -1761 +1965 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncap2 : ncap2 error"
-      IGCM_debug_Exit "ncap2"
+    echo "IGCM_sys_ncap2 : ncap2 error"
+    IGCM_debug_Exit "ncap2"
   fi
 
@@ -1768 +1972 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncatted
+#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncatted {
   IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
@@ -1798 +2007 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncatted : ncatted error"
-      IGCM_debug_Exit "ncatted"
+    echo "IGCM_sys_ncatted : ncatted error"
+    IGCM_debug_Exit "ncatted"
   fi
 
@@ -1805 +2014 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncbo
+#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncbo {
   IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
@@ -1835 +2049 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncbo : ncbo error"
-      IGCM_debug_Exit "ncbo"
+    echo "IGCM_sys_ncbo : ncbo error"
+    IGCM_debug_Exit "ncbo"
   fi
 
@@ -1842 +2056 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncdif
+#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncdiff {
   IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
@@ -1872 +2091 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncdiff : ncdiff error"
-      IGCM_debug_Exit "ncdiff"
+    echo "IGCM_sys_ncdiff : ncdiff error"
+    IGCM_debug_Exit "ncdiff"
   fi
 
@@ -1879 +2098 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncea
+#D-* Purpose: encapsulate ncea call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncea {
   IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
@@ -1909 +2133 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncea : ncea error"
-      IGCM_debug_Exit "ncea"
+    echo "IGCM_sys_ncea : ncea error"
+    IGCM_debug_Exit "ncea"
   fi
 
@@ -1916 +2140 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncecat
+#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncecat {
   IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
@@ -1946 +2175 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncecat : ncecat error"
-      IGCM_debug_Exit "ncecat"
+    echo "IGCM_sys_ncecat : ncecat error"
+    IGCM_debug_Exit "ncecat"
   fi
 
@@ -1953 +2182 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncflint
+#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncflint {
   IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
@@ -1983 +2217 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncflint : ncflint error"
-      IGCM_debug_Exit "ncflint"
+    echo "IGCM_sys_ncflint : ncflint error"
+    IGCM_debug_Exit "ncflint"
   fi
 
@@ -1990 +2224 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncks
+#D-* Purpose: encapsulate ncks call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncks {
   IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
@@ -2020 +2259 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncks : ncks error"
-      IGCM_debug_Exit "ncks"
+    echo "IGCM_sys_ncks : ncks error"
+    IGCM_debug_Exit "ncks"
   fi
 
@@ -2027 +2266 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncpdq
+#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncpdq {
   IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
@@ -2057 +2301 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncpdq : ncpdq error"
-      IGCM_debug_Exit "ncpdq"
+    echo "IGCM_sys_ncpdq : ncpdq error"
+    IGCM_debug_Exit "ncpdq"
   fi
 
@@ -2064 +2308 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncra
+#D-* Purpose: encapsulate ncra call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncra {
   IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
@@ -2094 +2343 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncra : ncra error"
-      IGCM_debug_Exit "ncra"
+    echo "IGCM_sys_ncra : ncra error"
+    IGCM_debug_Exit "ncra"
   fi
 
@@ -2101 +2350 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncrcat
+#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncrcat {
   IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
@@ -2107 +2361 @@
   fi
 
-  typeset NB_ESSAI DELAI status i
+  typeset NB_ESSAI DELAI status i lastArg
   # number of tentative
   NB_ESSAI=3
@@ -2123 +2377 @@
       IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
       sleep $DELAI
+    elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" out_rsync )" = "X" ] ; then
+      IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
+      cat out_rsync
+      # remove files having corrupted time axis
+      eval lastArg=\${$#}
+      IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
+      \rm ${lastArg}
+      \rm out_rsync
+      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
+      sleep $DELAI
     else
       \rm out_rsync
@@ -2131 +2395 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncrcat : ncrcat error"
-      #IGCM_debug_Exit "ncrcat"
+    echo "IGCM_sys_ncrcat : ncrcat error"
+    #IGCM_debug_Exit "ncrcat"
   fi
 
@@ -2138 +2402 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncrename
+#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncrename {
   IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
@@ -2168 +2437 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncrename : ncrename error"
-      IGCM_debug_Exit "ncrename"
+    echo "IGCM_sys_ncrename : ncrename error"
+    IGCM_debug_Exit "ncrename"
   fi
 
@@ -2175 +2444 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncwa
+#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncwa {
   IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
@@ -2205 +2479 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncwa : ncwa error"
-      IGCM_debug_Exit "ncwa"
+    echo "IGCM_sys_ncwa : ncwa error"
+    IGCM_debug_Exit "ncwa"
   fi
 
@@ -2215 +2489 @@
 # CDO OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_cdo
+#D-* Purpose: encapsulate cdo call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_cdo {
   IGCM_debug_PushStack "IGCM_sys_cdo" -- $@

trunk/libIGCM/libIGCM_sys/libIGCM_sys_iitm.ksh

@@ -210 +210 @@
 function IGCM_sys_RshArchive {
   IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
-  DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
-  if [ $? = 0 ] ; then
-    /bin/ksh <<-EOF
+  /bin/ksh <<-EOF
     ${@}
 EOF
-    status=$?
-  else
-    /bin/ksh <<-EOF
-    ${@}
-EOF
-    status=$?
-  fi
+  status=$?
   if [ ${status} -gt 0 ] ; then
     IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
@@ -547 +539 @@
   fi
 
-  \ls -r $@
+  \tree -f $@
 
   IGCM_debug_PopStack "IGCM_sys_Tree"
@@ -596 +588 @@
   IGCM_debug_PushStack "IGCM_sys_Qsub" $@
 
-  typeset ERROR
-  ERROR=0
+  typeset status
 
   if ( $DEBUG_sys ) ; then
@@ -606 +597 @@
   [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
   sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Output}:" \
-      -e "s:\# \@ error *= .*:\# \@ error = ${Script_Output}:"   \
-      $1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
-  cd $REP_FOR_JOB ; /usr/lpp/LoadL/full/bin/llsubmit JOB_FOR_IGCM ; ERROR=$? ; cd - ;
-
-  if [ ${ERROR} -gt 0 ] ; then
-    echo "IGCM_sys_Qsub : erreur $@"
+    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Output}:"   \
+    $1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
+  cd $REP_FOR_JOB ; /usr/lpp/LoadL/full/bin/llsubmit JOB_FOR_IGCM ; status=$? ; cd - ;
+
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
     IGCM_debug_Exit "IGCM_sys_Qsub"
   else
@@ -627 +618 @@
   IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
 
-  typeset ERROR
-  ERROR=0
+  typeset status
 
   if ( $DEBUG_sys ) ; then
@@ -638 +628 @@
 
   sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Post_Output}.out:" \
-      -e "s:\# \@ error *= .*:\# \@ error = ${Script_Post_Output}.out:"   \
-      ${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM
-
-  cd $REP_FOR_JOB ; /usr/lpp/LoadL/full/bin/llsubmit JOB_FOR_IGCM ; ERROR=$? ; cd - ;
-  if [ ${ERROR} -gt 0 ] ; then
-    echo "IGCM_sys_QsubPost : erreur $@."
+    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Post_Output}.out:"   \
+    ${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM
+
+  cd $REP_FOR_JOB ; /usr/lpp/LoadL/full/bin/llsubmit JOB_FOR_IGCM ; status=$? ; cd - ;
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
     IGCM_debug_Exit "IGCM_sys_QsubPost"
   else
     IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
   fi
-
   IGCM_debug_PopStack "IGCM_sys_QsubPost"
 }
@@ -872 +861 @@
     \rm out_rsync
   fi
-
   IGCM_debug_PopStack "IGCM_sys_RmRunDir"
 }
@@ -967 +955 @@
     typeset status
 
-      #USUAL WAY
-    \cp -rp $1 $2 > out_rsync 2>&1
+    #USUAL WAY
+    \cp -ur $1 $2 > out_rsync 2>&1
     status=$?
 
@@ -1158 +1146 @@
     #
     IGCM_sys_MkdirArchive $( dirname $2 )
-    #
-    typeset RET
     #
     if [ X${JobType} = XRUN ] ; then
@@ -1312 +1298 @@
       return
     fi
-
-    # SD : dm_liste is not suited for computing job
-    #      because we change filename during transfert
-    #      dm_liste is better suited for post-treatment
-    #dm_liste=" "
-    #(( ifile=1 ))
-    #while [ $ifile -lt $# ] ; do
-    #    dm_liste=$( eval echo ${dm_liste} " "\${${ifile}} )
-    #    (( ifile = ifile + 1 ))
-    #done
-    #DEST=$( eval echo \${${#}} )
 
     #USUAL WAY
@@ -1453 +1428 @@
 
 #D-#==================================================
+#D-function IGCM_sys_Dods_Rm
+#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
+#D-* Examples:
+#D-
+function IGCM_sys_Dods_Rm {
+  if ( $DEBUG_sys ) ; then
+    echo "IGCM_sys_Dods_Rm :" $@
+  fi
+  return 0
+}
+
+#D-#==================================================
+#D-function IGCM_sys_Dods_Cp
+#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
+#D-* Examples:
+#D-
+function IGCM_sys_Dods_Cp {
+  if ( $DEBUG_sys ) ; then
+    echo "IGCM_sys_Dods_Cp :" $@
+  fi
+  return 0
+}
+
+#D-#==================================================
 #D-function IGCM_sys_Put_Dods
 #D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
@@ -1619 +1618 @@
   IGCM_debug_PushStack "IGCM_sys_activ_variables"
   if ( $DEBUG_sys ) ; then
-    echo "IGCM_sys_activ_variables "
+    echo "IGCM_sys_activ_variables"
   fi
 
@@ -1644 +1643 @@
   IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
   if ( $DEBUG_sys ) ; then
-    echo "IGCM_sys_desactiv_variables "
+    echo "IGCM_sys_desactiv_variables"
   fi
 # --------------------------------------------------------------------
@@ -1667 +1666 @@
 function IGCM_sys_build_run_file {
 
-IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
+  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
 
 }
@@ -1831 +1830 @@
   fi
 
-  IGCM_debug_Print 1 "sys iitm : execution command is "
+  IGCM_debug_Print 1 "sys iitm : execution command is"
   IGCM_debug_Print 1 "$EXECUTION"
 
@@ -1911 +1910 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncap2 : ncap2 error"
-      IGCM_debug_Exit "ncap2"
+    echo "IGCM_sys_ncap2 : ncap2 error"
+    IGCM_debug_Exit "ncap2"
   fi
 
@@ -1953 +1952 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncatted : ncatted error"
-      IGCM_debug_Exit "ncatted"
+    echo "IGCM_sys_ncatted : ncatted error"
+    IGCM_debug_Exit "ncatted"
   fi
 
@@ -1995 +1994 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncbo : ncbo error"
-      IGCM_debug_Exit "ncbo"
+    echo "IGCM_sys_ncbo : ncbo error"
+    IGCM_debug_Exit "ncbo"
   fi
 
@@ -2037 +2036 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncdiff : ncdiff error"
-      IGCM_debug_Exit "ncdiff"
+    echo "IGCM_sys_ncdiff : ncdiff error"
+    IGCM_debug_Exit "ncdiff"
   fi
 
@@ -2079 +2078 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncea : ncea error"
-      IGCM_debug_Exit "ncea"
+    echo "IGCM_sys_ncea : ncea error"
+    IGCM_debug_Exit "ncea"
   fi
 
@@ -2121 +2120 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncecat : ncecat error"
-      IGCM_debug_Exit "ncecat"
+    echo "IGCM_sys_ncecat : ncecat error"
+    IGCM_debug_Exit "ncecat"
   fi
 
@@ -2163 +2162 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncflint : ncflint error"
-      IGCM_debug_Exit "ncflint"
+    echo "IGCM_sys_ncflint : ncflint error"
+    IGCM_debug_Exit "ncflint"
   fi
 
@@ -2205 +2204 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncks : ncks error"
-      IGCM_debug_Exit "ncks"
+    echo "IGCM_sys_ncks : ncks error"
+    IGCM_debug_Exit "ncks"
   fi
 
@@ -2247 +2246 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncpdq : ncpdq error"
-      IGCM_debug_Exit "ncpdq"
+    echo "IGCM_sys_ncpdq : ncpdq error"
+    IGCM_debug_Exit "ncpdq"
   fi
 
@@ -2289 +2288 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncra : ncra error"
-      IGCM_debug_Exit "ncra"
+    echo "IGCM_sys_ncra : ncra error"
+    IGCM_debug_Exit "ncra"
   fi
 
@@ -2341 +2340 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncrcat : ncrcat error"
+    echo "IGCM_sys_ncrcat : ncrcat error"
       #IGCM_debug_Exit "ncrcat"
   fi
@@ -2383 +2382 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncrename : ncrename error"
-      IGCM_debug_Exit "ncrename"
+    echo "IGCM_sys_ncrename : ncrename error"
+    IGCM_debug_Exit "ncrename"
   fi
 
@@ -2425 +2424 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncwa : ncwa error"
-      IGCM_debug_Exit "ncwa"
+    echo "IGCM_sys_ncwa : ncwa error"
+    IGCM_debug_Exit "ncwa"
   fi
 

trunk/libIGCM/libIGCM_sys/libIGCM_sys_obelix.ksh

@@ -123 +123 @@
 
 #====================================================
-#- ARCHIVE (dedicated to large files)
+#- ARCHIVE
 typeset ARCHIVE=${ARCHIVE:=/home/scratch01/${LOGIN}}
 
@@ -173 +173 @@
 #D-
 function IGCM_sys_ChangeArchive {
-
   IGCM_debug_Print 1 " dummy function : IGCM_sys_ChangeArchive "
-
 }
 
@@ -209 +207 @@
     ${@}
 EOF
-  if [ $? -gt 0 ] ; then
-    echo "IGCM_sys_RshArchive : erreur."
+  status=$?
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
     IGCM_debug_Exit "IGCM_sys_RshArchive"
   fi
@@ -227 +226 @@
   fi
 
-  #echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
-  #cat tmp_IGCM_sys_RshPost_$$
   # keep standard input (stdin) for the loop onto temporary file
   cat >tmp_IGCM_sys_RshPost_$$
@@ -234 +231 @@
 # ============ FRONTEND START ============ #
 
-    ssh obelix3 exec /bin/ksh <tmp_IGCM_sys_RshPost_$$
+  ssh obelix3 exec /bin/ksh <tmp_IGCM_sys_RshPost_$$
   if [ $? -gt 0 ] ; then
     echo "IGCM_sys_RshPost : erreur."
@@ -279 +276 @@
   fi
 
+  sleep 10
+  rm -f job_end.mail
+
   if [ $? -gt 0 ] ; then
     echo "IGCM_sys_SendMail : erreur."
@@ -323 +323 @@
   #- creation de repertoire sur le serveur fichier
   if [ ! -d ${1} ]; then
-    \mkdir -p $1
-    if [ $? -gt 0 ] ; then
-      echo "IGCM_sys_MkdirArchive : erreur."
+    mkdir -p $1
+    status=$?
+
+    if [ ${status} -gt 0 ] ; then
+      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
       IGCM_debug_Exit "IGCM_sys_MkdirArchive"
     fi
@@ -437 +439 @@
   ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
   IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
-
   return ${ExistFlag}
 }
@@ -577 +578 @@
     echo "IGCM_sys_Qsub :" $@
   fi
+  typeset status
   /usr/local/bin/qsub -q medium -o ${Script_Output} -N ${config_UserChoices_JobName}.${CumulPeriod} < $1
-  if [ $? -gt 0 ] ; then
-    echo "IGCM_sys_Qsub : erreur."
+  status=$?
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
     IGCM_debug_Exit "IGCM_sys_Qsub"
   fi
@@ -595 +598 @@
     echo "IGCM_sys_QsubPost :" $@
   fi
+  typeset status
   /usr/local/bin/qsub -q medium -o ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job -v ${listVarEnv}
-  if [ $? -gt 0 ] ; then
-    echo "IGCM_sys_QsubPost : erreur " $@
+  status=$?
+  if [ ${status} -gt 0 ] ; then
+    IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
     IGCM_debug_Exit "IGCM_sys_QsubPost"
   fi
@@ -719 +724 @@
 
 #D-#==================================================
+#D-function IGCM_sys_Miror_libIGCM
+#D-* Purpose: Mirror libIGCM PATH and lib to frontend
+#D-* Examples:
+#D-
+function IGCM_sys_Mirror_libIGCM {
+  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
+  if ( $DEBUG_sys ) ; then
+    echo "IGCM_sys_Mirror_libIGCM"
+  fi
+
+  typeset status
+
+  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
+
+  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
+  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
+  status=$?
+
+  if [ ${status} -gt 0 ] ; then
+    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
+    cat out_rsync
+  fi
+  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
+}
+
+#D-#==================================================
 #D-function IGCM_sys_Cp
 #D-* Purpose: generic cp
@@ -791 +822 @@
 
   if [ ${status} -gt 0 ] ; then
-    echo "IGCM_sys_RmRunDir : error code ${status}"
+    echo "IGCM_sys_RmRunDir : rm error code is ${status}."
     cat out_rsync
     IGCM_debug_Exit "IGCM_sys_RmRunDir"
@@ -860 +891 @@
 
     if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_Put_Dir : error code ${status}"
+      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_Put_Dir"
@@ -892 +923 @@
 
     #USUAL WAY
-    \cp -R $1 $2 > out_rsync 2>&1
+    \cp -ur $1 $2 > out_rsync 2>&1
     status=$?
 
     if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_Get_Dir : error."
+      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_Get_Dir"
@@ -925 +956 @@
     fi
 
-    typeset status
-
-    #USUAL WAY
-    cp -RL $1 $2 > out_rsync 2>&1
-    status=$?
+    typeset NB_ESSAI DELAI status i
+    # number of tentative
+    NB_ESSAI=3
+    # time delay between tentative
+    DELAI=2
+
+    i=0
+    while [ $i -lt $NB_ESSAI ] ; do
+      \cp -urL $1 $2 > out_rsync 2>&1
+      status=$?
+      if [ ${status} -gt 0 ]; then
+        IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
+        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
+        sleep $DELAI
+      else
+        break
+      fi
+      (( i = i + 1 ))
+    done
 
     if [ ${status} -gt 0 ] ; then
@@ -936 +981 @@
       IGCM_debug_Exit "IGCM_sys_Get_Master"
     else
-      rm out_rsync
+      \rm out_rsync
     fi
   else
@@ -943 +988 @@
   IGCM_debug_PopStack "IGCM_sys_Get_Master"
 }
+
+#====================================================
+#- Call IGCM_sys_Mirror_libIGCM now !
+if ( $MirrorlibIGCM ) ; then
+  IGCM_sys_Mirror_libIGCM
+fi
 
 #D-#==================================================
@@ -966 +1017 @@
       IGCM_sys_Chmod 444 ${1}
     fi
+    #
+    # Only if we use rsync
+    #IGCM_sys_MkdirArchive $( dirname $2 )
+    #
+    #USUAL WAY
+    \cp $1 $2 > out_rsync 2>&1
+    status=$?
+
+#       #RSYNC WITH NETWORK SSH CALL
+#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
+#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
+
+#       #RSYNC WITH NFS USE
+#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
+#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
+
+#       status=$?
+#       IGCM_sys_Rsync_out $status
+
+#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
+#       (( status=status+$? ))
+
+    if [ ${status} -gt 0 ] ; then
+      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
+      cat out_rsync
+      IGCM_debug_Exit "IGCM_sys_Put_Rest"
+    else
+      \rm out_rsync
+    fi
+  else
+    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
+  fi
+  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
+}
+
+#D-#==================================================
+#D-function IGCM_sys_PutBuffer_Rest
+#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
+#D-           File and target directory must exist.
+#D-* Examples:
+#D-
+function IGCM_sys_PutBuffer_Rest {
+  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
+  if ( $DEBUG_sys ) ; then
+    echo "IGCM_sys_PutBuffer_Rest :" $@
+  fi
+  if [ $DRYRUN = 0 ]; then
+    if [ ! -f ${1} ] ; then
+      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
+      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
+    fi
+
+    typeset status
+    #
+    if [ X${JobType} = XRUN ] ; then
+      IGCM_sys_Chmod 444 ${1}
+    fi
 
     #
@@ -972 +1080 @@
     status=$?
 
-#       #RSYNC WITH NETWORK SSH CALL
-#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
-#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
-
-#       #RSYNC WITH NFS USE
-#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
-#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
-
-#       status=$?
-#       IGCM_sys_Rsync_out $status
-
-#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
-#       (( status=status+$? ))
-
-    if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
-      cat out_rsync
-      IGCM_debug_Exit "IGCM_sys_Put_Rest"
-    else
-      \rm out_rsync
-    fi
-  else
-    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
-  fi
-  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
-}
-
-#D-#==================================================
-#D-function IGCM_sys_PutBuffer_Rest
-#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
-#D-           File and target directory must exist.
-#D-* Examples:
-#D-
-function IGCM_sys_PutBuffer_Rest {
-  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
-  if ( $DEBUG_sys ) ; then
-    echo "IGCM_sys_PutBuffer_Rest :" $@
-  fi
-  if [ $DRYRUN = 0 ]; then
-    if [ ! -f ${1} ] ; then
-      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
-      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
-    fi
-
-    typeset status
-    #
-    if [ X${JobType} = XRUN ] ; then
-      IGCM_sys_Chmod 444 ${1}
-    fi
-
-    #
-    # USUAL WAY
-    \cp $1 $2 > out_rsync 2>&1
-    status=$?
-
     if [ ${status} -gt 0 ] ; then
       echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
+      [ -f ${2} ] && ls -l ${2}
+      [ -f ${2}/${1} ] && ls -l ${2}/${1}
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
@@ -1061 +1116 @@
     echo "IGCM_sys_Put_Out :" $@
   fi
+
+  typeset status
+
   if [ $DRYRUN = 0 ]; then
-    if [ -f ${1} ] ; then
-      if [ ! -d $( dirname $2 ) ] ; then
-        IGCM_sys_MkdirArchive $( dirname $2 )
-      fi
-    else
+    if [ ! -f ${1} ] ; then
       echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
       IGCM_debug_PopStack "IGCM_sys_Put_Out"
       return 1
     fi
-
-    typeset status
+    #
+    IGCM_sys_MkdirArchive $( dirname $2 )
     #
     if [ X${JobType} = XRUN ] ; then
@@ -1090 +1144 @@
 
     if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_Put_Out : error."
+      IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_Put_Out"
@@ -1114 +1168 @@
   fi
 
-  typeset status
+  typeset NB_ESSAI DELAI status i exist skip
+
+  # number of tentative
+  NB_ESSAI=3
+  # time delay between tentative
+  DELAI=2
 
   if [ $DRYRUN = 0 ]; then
@@ -1126 +1185 @@
     #
 
-    if [ X${JobType} = XRUN ] ; then
-      if [ X${3} = X ] ; then
-        IGCM_sys_Chmod 444 ${1}
+    exist=false
+    skip=false
+    if [ -f $2 ] ; then
+      IGCM_debug_Print 1 "$2 already exist"
+      exist=true
+      if [ "X$( diff $1 $2 )" = X ] ; then
+        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
+        status=0
+        skip=true
+      else
+        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
+        skip=false
       fi
     fi
     #
-    # USUAL WAY
-    \cp $1 $2 > out_rsync 2>&1
-    status=$?
+    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
+      IGCM_sys_Chmod u+w $2
+    fi
+
+    if [ X${skip} = Xfalse ] ; then
+      i=0
+      while [ $i -lt $NB_ESSAI ] ; do
+        # USUAL WAY
+        \cp $1 $2 > out_rsync 2>&1
+        status=$?
+        if [ ${status} -gt 0 ]; then
+          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
+          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
+          [ -f ${2} ] && ls -l ${2}
+          [ -f ${2}/${1} ] && ls -l ${2}/${1}
+          sleep $DELAI
+        else
+          break
+        fi
+        (( i = i + 1 ))
+      done
+    fi
 
     if [ ${status} -gt 0 ] ; then
       echo "IGCM_sys_PutBuffer_Out : error."
+      [ -f ${2} ] && ls -l ${2}
+      [ -f ${2}/${1} ] && ls -l ${2}/${1}
       cat out_rsync
       IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
     else
+
+      if [ X${JobType} = XRUN ] ; then
+        if [ X${3} = X ] ; then
+          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
+          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
+        fi
+      fi
+
       \rm out_rsync
     fi
@@ -1158 +1255 @@
   IGCM_debug_PushStack "IGCM_sys_Get" $@
 
-  typeset DEST RET status dm_liste ifile target
+  typeset DEST status dm_liste
 
   if ( $DEBUG_sys ) ; then
@@ -1178 +1275 @@
       echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
       IGCM_debug_Exit "IGCM_sys_Get"
-      #return
-    fi
-
-    (( RET=0 ))
-    for target in ${dm_liste[*]} ; do
-      \cp ${target} ${DEST} >> out_rsync 2>&1
-      (( RET=RET+$? ))
-    done
-
-#       echo ${RSYNC} ${RSYNC_opt} $@ > out_rsync 2>&1
-#       ${RSYNC} ${RSYNC_opt} $@ >> out_rsync 2>&1
-#       status=$?
-#       IGCM_sys_Rsync_out $status
-
-#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
-#       (( RET=RET+$? ))
-
-    if [ ${RET} -gt 0 ] ; then
-      echo "IGCM_sys_Get : error"
+      return
+    fi
+
+    #USUAL WAY
+    \cp ${dm_liste[*]} ${DEST} > out_rsync 2>&1
+    status=$?
+
+    if [ ${status} -gt 0 ] ; then
+      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
       cat out_rsync
-      \rm out_rsync
       IGCM_debug_Exit "IGCM_sys_Get"
     else
@@ -1217 +1303 @@
   IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
 
-  typeset DEST RET status buf_liste ifile target file_work
+  typeset DEST buf_liste target file_work
+  typeset NB_ESSAI DELAI status i
 
   if ( $DEBUG_sys ) ; then
     echo "IGCM_sys_GetBuffer :" $@
   fi
+
+  # number of tentative
+  NB_ESSAI=3
+  # time delay between tentative
+  DELAI=2
+
   if [ $DRYRUN -le 2 ]; then
     if [ X${1} = X'/l' ] ; then
@@ -1233 +1326 @@
     #USUAL WAY
     if [ X${1} = X'/l' ] ; then
-      (( RET=0 ))
       for target in ${buf_liste[*]} ; do
         local_file=$( basename ${target} )
-        \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
-        (( RET = RET + $? ))
+        i=0
+        while [ $i -lt $NB_ESSAI ] ; do
+          \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
+          status=$?
+          if [ ${status} -gt 0 ]; then
+            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
+            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
+            sleep $DELAI
+          else
+            break
+          fi
+          (( i = i + 1 ))
+        done
+        if [ ${status} -gt 0 ] ; then
+          echo "IGCM_sys_Get : error"
+          cat out_rsync
+          \rm out_rsync
+          IGCM_debug_Exit "IGCM_sys_GetBuffer"
+        else
+          \rm out_rsync
+        fi
       done
     else
-      \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
-      status=$?
-    fi
-
-    if [ ${RET} -gt 0 ] ; then
-      echo "IGCM_sys_GetBuffer : error"
-      cat out_rsync
-      \rm out_rsync
-      IGCM_debug_Exit "IGCM_sys_GetBuffer"
-    else
-      \rm out_rsync
+      i=0
+      while [ $i -lt $NB_ESSAI ] ; do
+        \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
+        status=$?
+        if [ ${status} -gt 0 ]; then
+          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
+          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
+          sleep $DELAI
+        else
+          break
+        fi
+        (( i = i + 1 ))
+      done
+      if [ ${status} -gt 0 ] ; then
+        echo "IGCM_sys_Get : error"
+        cat out_rsync
+        \rm out_rsync
+        IGCM_debug_Exit "IGCM_sys_GetBuffer"
+      else
+        \rm out_rsync
+      fi
     fi
   else
@@ -1319 +1440 @@
 #D-#==================================================
 #D-function IGCM_sys_Put_Dods
-#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
+#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
 #D-* Examples:
 #D-
@@ -1333 +1454 @@
 # REBUILD OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_rebuild
+#D-* Purpose: rebuild parallel files
+#D-* Examples:
+#D-
 function IGCM_sys_rebuild {
   IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
@@ -1339 +1465 @@
   fi
 
-  typeset status
-
-  /home/users/igcmg/rebuild/bin/rebuild -f -o $@
-  status=$?
+  typeset NB_ESSAI DELAI status i firstArg
+  # number of tentative
+  NB_ESSAI=3
+  # time delay between tentative
+  DELAI=2
+
+  i=0
+  while [ $i -lt $NB_ESSAI ] ; do
+    /home/users/igcmg/rebuild/bin/rebuild -f -o $@ > out_rsync 2>&1
+    status=$?
+    if [ ${status} -gt 0 ] ; then
+      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
+      cat out_rsync
+      \rm out_rsync
+      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
+      firstArg=${1}
+      \rm ${firstArg}
+      sleep $DELAI
+    else
+      \rm out_rsync
+      break
+    fi
+    (( i = i + 1 ))
+  done
+
   if [ ${status} -gt 0 ] ; then
-    IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
+    echo "IGCM_sys_rebuild : rebuild error code is ${status}"
     IGCM_debug_Exit "rebuild"
   fi
@@ -1351 +1498 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_rebuild_station
+#D-* Purpose: rebuild parallel files describing station 
+#D-* Examples:
+#D-
 function IGCM_sys_rebuild_station {
   IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
@@ -1445 +1597 @@
 # Activate Running Environnment Variables
 
+#D-#==================================================
+#D-function IGCM_sys_desactiv_variables
+#D-* Purpose: set environement variables prior to execution
+#D-* Examples:
+#D-
 function IGCM_sys_activ_variables {
   IGCM_debug_PushStack "IGCM_sys_activ_variables"
@@ -1450 +1607 @@
     echo "IGCM_sys_activ_variables"
   fi
+
+# --------------------------------------------------------------------
+#D- MPI specifications
+# --------------------------------------------------------------------
+
+# --------------------------------------------------------------------
+#D- Other specifications
+# --------------------------------------------------------------------
+
   IGCM_debug_PopStack "IGCM_sys_activ_variables"
 }
@@ -1456 +1622 @@
 # Desactivate Running Environnment Variables
 
+#D-#==================================================
+#D-function IGCM_sys_desactiv_variables
+#D-* Purpose: unset environement variables after execution
+#D-* Examples:
+#D-
 function IGCM_sys_desactiv_variables {
   IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
@@ -1461 +1632 @@
     echo "IGCM_sys_desactiv_variables"
   fi
+# --------------------------------------------------------------------
+#D- MPI specifications
+# --------------------------------------------------------------------
+
+# --------------------------------------------------------------------
+#D- Other specifications
+# --------------------------------------------------------------------
+
   IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
 }
@@ -1467 +1646 @@
 # Build MPI/OMP scripts run file (dummy function)
 
+#D-#==================================================
+#D-function IGCM_sys_build_run_file
+#D-* Purpose: build run file (deprecated)
+#D-* Examples:
+#D-
 function IGCM_sys_build_run_file {
 
@@ -1475 +1659 @@
 ############################################################
 # Build MPI/OMP scripts
+
+#D-#==================================================
+#D-function IGCM_sys_build_execution_scripts
+#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
+#D-* Examples:
+#D-
 function IGCM_sys_build_execution_scripts
 {
@@ -1486 +1676 @@
 
   if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
-    IGCM_debug_Exit "IGCM_sys_titane build_execution_scripts : Job_${config_UserChoices_JobName} doesn't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
+    IGCM_debug_Exit "IGCM_sys_obelix build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
   fi
 
@@ -1675 +1865 @@
       done
 
-        # Then second loop on the components
+      # Then second loop on the components
 
       for comp in ${config_ListOfComponents[*]} ; do
@@ -1727 +1917 @@
   fi
 
-  IGCM_debug_Print 1 "sys Obelix : La commande d execution est "
-  IGCM_debug_Print 1 $EXECUTION
+  IGCM_debug_Print 1 "sys Obelix : execution command is"
+  IGCM_debug_Print 1 "$EXECUTION"
 
   IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
@@ -1734 +1924 @@
 
 ############################################################
-# Check of space available on temporary filesytems
+# Check of space available on temporary filesytems. Dummy function here
+
+#D-#==================================================
+#D-function IGCM_sys_check_quota. Dummy call here
+#D-* Purpose: check user quota. Stop the simulation if quota above 90%
+#D-* Examples:
+#D-
 function IGCM_sys_check_quota {
   IGCM_debug_PushStack "IGCM_sys_check_quota"
@@ -1743 +1939 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_CountJobInQueue
+#D-* Purpose: Check if job_name is currently
+#D-  running or in queue
+#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
+#D-
+function IGCM_sys_CountJobInQueue {
+  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
+  if ( $DEBUG_sys ) ; then
+    echo "IGCM_sys_CountJobInQueue"
+  fi
+  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
+}
+
 ##############################################################
 # NCO OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_ncap2
+#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncap2 {
   IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
@@ -1776 +1991 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncap2 : ncap2 error"
-      IGCM_debug_Exit "ncap2"
+    echo "IGCM_sys_ncap2 : ncap2 error"
+    IGCM_debug_Exit "ncap2"
   fi
 
@@ -1783 +1998 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncatted
+#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncatted {
   IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
@@ -1813 +2033 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncatted : ncatted error"
-      IGCM_debug_Exit "ncatted"
+    echo "IGCM_sys_ncatted : ncatted error"
+    IGCM_debug_Exit "ncatted"
   fi
 
@@ -1820 +2040 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncbo
+#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncbo {
   IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
@@ -1850 +2075 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncbo : ncbo error"
-      IGCM_debug_Exit "ncbo"
+    echo "IGCM_sys_ncbo : ncbo error"
+    IGCM_debug_Exit "ncbo"
   fi
 
@@ -1857 +2082 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncdif
+#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncdiff {
   IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
@@ -1887 +2117 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncdiff : ncdiff error"
-      IGCM_debug_Exit "ncdiff"
+    echo "IGCM_sys_ncdiff : ncdiff error"
+    IGCM_debug_Exit "ncdiff"
   fi
 
@@ -1894 +2124 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncea
+#D-* Purpose: encapsulate ncea call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncea {
   IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
@@ -1924 +2159 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncea : ncea error"
-      IGCM_debug_Exit "ncea"
+    echo "IGCM_sys_ncea : ncea error"
+    IGCM_debug_Exit "ncea"
   fi
 
@@ -1931 +2166 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncecat
+#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncecat {
   IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
@@ -1961 +2201 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncecat : ncecat error"
-      IGCM_debug_Exit "ncecat"
+    echo "IGCM_sys_ncecat : ncecat error"
+    IGCM_debug_Exit "ncecat"
   fi
 
@@ -1968 +2208 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncflint
+#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncflint {
   IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
@@ -1998 +2243 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncflint : ncflint error"
-      IGCM_debug_Exit "ncflint"
+    echo "IGCM_sys_ncflint : ncflint error"
+    IGCM_debug_Exit "ncflint"
   fi
 
@@ -2005 +2250 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncks
+#D-* Purpose: encapsulate ncks call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncks {
   IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
@@ -2035 +2285 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncks : ncks error"
-      IGCM_debug_Exit "ncks"
+    echo "IGCM_sys_ncks : ncks error"
+    IGCM_debug_Exit "ncks"
   fi
 
@@ -2042 +2292 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncpdq
+#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncpdq {
   IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
@@ -2072 +2327 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncpdq : ncpdq error"
-      IGCM_debug_Exit "ncpdq"
+    echo "IGCM_sys_ncpdq : ncpdq error"
+    IGCM_debug_Exit "ncpdq"
   fi
 
@@ -2079 +2334 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncra
+#D-* Purpose: encapsulate ncra call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncra {
   IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
@@ -2109 +2369 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncra : ncra error"
-      IGCM_debug_Exit "ncra"
+    echo "IGCM_sys_ncra : ncra error"
+    IGCM_debug_Exit "ncra"
   fi
 
@@ -2116 +2376 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncrcat
+#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncrcat {
   IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
@@ -2122 +2387 @@
   fi
 
-  typeset NB_ESSAI DELAI status i
+  typeset NB_ESSAI DELAI status i lastArg
   # number of tentative
   NB_ESSAI=3
@@ -2138 +2403 @@
       IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
       sleep $DELAI
+    elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" out_rsync )" = "X" ] ; then
+      IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
+      cat out_rsync
+      # remove files having corrupted time axis
+      eval lastArg=\${$#}
+      IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
+      \rm ${lastArg}
+      \rm out_rsync
+      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
+      sleep $DELAI
     else
       \rm out_rsync
@@ -2146 +2421 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncrcat : ncrcat error"
-      #IGCM_debug_Exit "ncrcat"
+    echo "IGCM_sys_ncrcat : ncrcat error"
+    #IGCM_debug_Exit "ncrcat"
   fi
 
@@ -2153 +2428 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncrename
+#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncrename {
   IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
@@ -2183 +2463 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncrename : ncrename error"
-      IGCM_debug_Exit "ncrename"
+    echo "IGCM_sys_ncrename : ncrename error"
+    IGCM_debug_Exit "ncrename"
   fi
 
@@ -2190 +2470 @@
 }
 
+#D-#==================================================
+#D-function IGCM_sys_ncwa
+#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_ncwa {
   IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
@@ -2220 +2505 @@
 
   if [ ${status} -gt 0 ] ; then
-      echo "IGCM_sys_ncwa : ncwa error"
-      IGCM_debug_Exit "ncwa"
+    echo "IGCM_sys_ncwa : ncwa error"
+    IGCM_debug_Exit "ncwa"
   fi
 
@@ -2230 +2515 @@
 # CDO OPERATOR
 
+#D-#==================================================
+#D-function IGCM_sys_cdo
+#D-* Purpose: encapsulate cdo call so as to manage error code and retry
+#D-* Examples:
+#D-
 function IGCM_sys_cdo {
   IGCM_debug_PushStack "IGCM_sys_cdo" -- $@