Changeset 565

Timestamp:

02/29/12 15:41:53 (12 years ago)

Author:

sdipsl

Message:

• Pack submission is active for pack_restart and pack_debug.
  • work on titane
  • SX9 needs a patch (post-processing will move to mercure frontend)
• Use IGCM_date function in pack_restart and pack_debug. Should work for any frequency.
• Cleaning.

Location:

trunk/libIGCM

Files:

• AA_pack_debug (modified) (7 diffs)
• AA_pack_restart (modified) (6 diffs)
• libIGCM_post/libIGCM_post.ksh (modified) (2 diffs)

trunk/libIGCM/AA_pack_debug

@@ -266 +266 @@
 #### Preparation de l'environnement
 # repertoire dans lequel seront crees les archives
+#STORE_DEBUG=/scratch/cont003/p86denv/Pack_Test/DEBUG
 STORE_DEBUG=${R_SAVE}/DEBUG
 IGCM_sys_MkdirArchive ${STORE_DEBUG}
-
-#STORE_DEBUG=/scratch/cont003/p86denv/Pack_Test/DEBUG
-#mkdir -p ${STORE_DEBUG}
 
 # Switch to script variables meaning (try to be compatible with ipsl_pack TGCC moving procedure)
@@ -290 +288 @@
 find ${PATH_SIMUL}/*/Debug -type f | sort >> ${RUN_DIR}/liste_debug_files.txt
 
-###cat  ${RUN_DIR}/liste_debug_files.txt | xargs dmget 
-
 # Rename debug files we will work with using symlinks
 #gawk -F/ '{dest_name=$(NF-2)("_") $(NF); system("cp --preserve " $0 " "dest_name)}' ${RUN_DIR}/liste_debug_files.txt
@@ -297 +293 @@
 
 # List renamed debug files
-ls -l ${RUN_DIR}/???_*${JobName}* > ${RUN_DIR}/new_liste_debug_files.txt
-
-### 2- creer les listes de fichiers par periode ####
-###Â 2.1- dechiffrer PeriodPack et le separer en year et month ###
-case ${PeriodPack} in
-*Y|*y) 
-  periodpackinyear=$( echo ${PeriodPack} | sed -e 's/[yY]//' ) 
-  periodpackinmonth=0
-  ;;
-
-*M|*m) 
-  periodpackinyear=0
-  periodpackinmonth=$( echo ${PeriodPack} | sed -e 's/[mM]//' ) 
-  ;;
-
-*)
-  echo "PeriodPack doit etre en mois ou annees"
-  IGCM_debug_Exit "period pack unit"
-  ;;
-
-esac
-
-### 2.2- extraire l'annee et le mois de DateBegin ###
-date_begin=${DateBegin}
-date_end=${DateEnd}
-
-IGCM_date_GetYearMonth ${date_begin} year_begin month_begin
-IGCM_date_GetYearMonth ${date_end}   year_end   month_end
+find ${RUN_DIR} -name "???_*${JobName}*" -ls | sort -k 11 > ${RUN_DIR}/new_liste_debug_files.txt
 
 # ------------------------------------------------------------------
@@ -331 +300 @@
 IGCM_debug_Verif_Exit_Post
 
-### 2.3- creation des listes ###
-date_begin_pack=${date_begin}
-month_begin_pack=${month_begin}
-year_begin_pack=${year_begin}
-
-date_end_simulation=${date_end}
-
+# Init loop
+date_begin_pack=${DateBegin}
+date_end_simulation=${DateEnd}
 number_pack=1
 
@@ -347 +312 @@
   IGCM_debug_PrintVariables 3 number_pack
 
-  (( month_end_pack = month_begin_pack + periodpackinmonth ))
-  (( add_year = ${month_end_pack} / 13 ))
-  if [ ${month_end_pack} -ge 13 ] ; then 
-    (( month_end_pack = ${month_end_pack} % 12 ))
-  fi 
-
-  (( year_end_pack = year_begin_pack + periodpackinyear + add_year ))
-
-  date_end_pack=${year_end_pack}$( printf "%02i\n" ${month_end_pack} )01
-
-  for file in $( gawk '{print $9}' ${RUN_DIR}/new_liste_debug_files.txt ); do
+  DaysTemp=$( IGCM_date_DaysInNextPeriod ${date_begin_pack} ${PeriodPack} )
+  date_end_pack=$( IGCM_date_AddDaysToGregorianDate ${date_begin_pack} ${DaysTemp} )
+
+  for file in $( gawk '{print $11}' ${RUN_DIR}/new_liste_debug_files.txt ); do
     extract_date_file=$( echo ${file}  | sed -e "s/.*${JobName}_[0-9]*_//" )
     date_file=$( echo ${extract_date_file} | sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
@@ -363 +321 @@
     if [ ${date_file} -lt ${date_end_pack} ] && [ ${date_file} -ge ${date_begin_pack} ] ; then 
         echo $( basename ${file} ) >> ${RUN_DIR}/liste_pack_${number_pack}.txt
+        echo $( ls -L ${file} ) >> ${RUN_DIR}/liste_remove_pack_${number_pack}.txt
+
 #    Will not work. Need to find a better ordering.
 #    elif [ ${date_file} -ge ${date_end_pack} ] ; then
@@ -369 +329 @@
   done
 
-  month_begin_pack=${month_end_pack}
-  year_begin_pack=${year_end_pack}
-  date_begin_pack=${year_begin_pack}$( printf "%02i\n" ${month_begin_pack} )01
-
-  # Here we create tar files directly in $STORE. Avoid to create then to copy.
-  tar -cvf ${STORE_DEBUG}/${JobName}_${year_begin_pack}_${year_end_pack}_debug.tar --dereference --files-from ${RUN_DIR}/liste_pack_${number_pack}.txt
-  cat  ${RUN_DIR}/liste_pack_${number_pack}.txt | xargs rm 
+  # Create tar files
+  IGCM_debug_Print 1 "Tar ongoing"
+  tar -cvf ${JobName}_${date_begin_pack}_${date_end_pack}_debug.tar --dereference --files-from ${RUN_DIR}/liste_pack_${number_pack}.txt
+  # Save it
+  IGCM_sys_Put_Out ${JobName}_${date_begin_pack}_${date_end_pack}_debug.tar ${STORE_DEBUG}
+  # Clean it
+  IGCM_sys_Rm ${JobName}_${date_begin_pack}_${date_end_pack}_debug.tar
+  # Clean links
+  cat ${RUN_DIR}/liste_pack_${number_pack}.txt | xargs rm
+  # Clean files links pointed to
+  cat ${RUN_DIR}/liste_remove_pack_${number_pack}.txt | xargs rm
+  # Save the list of files that has been pack (tar)
   mv ${RUN_DIR}/liste_pack_${number_pack}.txt ${STORE_DEBUG}
+  IGCM_debug_Print 1 "Tar and cleaning done"
 
   (( number_pack = number_pack + 1 ))
 
+  # Add 1 day to date_end_pack to have the new date_begin_pack
+  date_begin_pack=$( $( IGCM_date_AddDaysToGregorianDate ${date_end_pack} 1 )
+
 done

trunk/libIGCM/AA_pack_restart

@@ -266 +266 @@
 #### Preparation de l'environnement
 # repertoire dans lequel seront crees les archives
+#STORE_RESTART=/scratch/cont003/p86denv/Pack_Test/RESTART
 STORE_RESTART=${R_SAVE}/RESTART
 IGCM_sys_MkdirArchive ${STORE_RESTART}
-
-#STORE_RESTART=/scratch/cont003/p86denv/Pack_Test/RESTART
-#mkdir -p ${STORE_RESTART}
 
 # Switch to script variables meaning (try to be compatible with ipsl_pack TGCC moving procedure)
@@ -289 +287 @@
 # Create sorted restart list to work with
 find ${PATH_SIMUL}/*/Restart -name "*nc" | sort >> ${RUN_DIR}/liste_restart_files.txt
-#sort ${RUN_DIR}/liste_restart_files.txt > ${RUN_DIR}/tmp.txt
-#mv ${RUN_DIR}/tmp.txt ${RUN_DIR}/liste_restart_files.txt
-
-###cat  ${RUN_DIR}/liste_restart_files.txt | xargs dmget 
-
-# Rename restart files we will work with
+
+# Rename restart files we will work with using symlinks
 #gawk -F/ '{dest_name=$(NF-2)("_") $(NF); system("cp --preserve " $0 " "dest_name)}' ${RUN_DIR}/liste_restart_files.txt
 gawk -F/ '{dest_name=$(NF-2)("_") $(NF); system("ln -s " $0 " "dest_name)}' ${RUN_DIR}/liste_restart_files.txt
 
 # List renamed restart files
-ls -l ${RUN_DIR}/*.nc > ${RUN_DIR}/new_liste_restart_files.txt
-
-### 2- creer les listes de fichiers par periode ####
-###Â 2.1- dechiffrer PeriodPack et le separer en year et month ###
-case ${PeriodPack} in
-*Y|*y) 
-  periodpackinyear=$( echo ${PeriodPack} | sed -e 's/[yY]//' ) 
-  periodpackinmonth=0
-  ;;
-
-*M|*m) 
-  periodpackinyear=0
-  periodpackinmonth=$( echo ${PeriodPack} | sed -e 's/[mM]//' ) 
-  ;;
-
-*)
-  echo "PeriodPack doit etre en mois ou annees"
-  IGCM_debug_Exit "period pack unit"
-  ;;
-
-esac
-
-### 2.2- extraire l'annee et le mois de DateBegin ###
-date_begin=${DateBegin}
-date_end=${DateEnd}
-
-IGCM_date_GetYearMonth ${date_begin} year_begin month_begin
-IGCM_date_GetYearMonth ${date_end}   year_end   month_end
+find ${RUN_DIR} -name "*.nc" -ls | sort -k 11 > ${RUN_DIR}/new_liste_restart_files.txt
 
 # ------------------------------------------------------------------
@@ -333 +300 @@
 IGCM_debug_Verif_Exit_Post
 
-### 2.3- creation des listes ###
-date_begin_pack=${date_begin}
-month_begin_pack=${month_begin}
-year_begin_pack=${year_begin}
-
-date_end_simulation=${date_end}
-
+# Init loop
+date_begin_pack=${DateBegin}
+date_end_simulation=${DateEnd}
 number_pack=1
 
@@ -349 +312 @@
   IGCM_debug_PrintVariables 3 number_pack
 
-  (( month_end_pack = month_begin_pack + periodpackinmonth ))
-  (( add_year = ${month_end_pack} / 13 ))
-  if [ ${month_end_pack} -ge 13 ] ; then 
-    (( month_end_pack = ${month_end_pack} % 12 ))
-  fi 
-
-  (( year_end_pack = year_begin_pack + periodpackinyear + add_year ))
-
-  date_end_pack=${year_end_pack}$( printf "%02i\n" ${month_end_pack} )01
-
-  for file in $( gawk '{print $9}' ${RUN_DIR}/new_liste_restart_files.txt ); do
+  DaysTemp=$( IGCM_date_DaysInNextPeriod ${date_begin_pack} ${PeriodPack} )
+  date_end_pack=$( IGCM_date_AddDaysToGregorianDate ${date_begin_pack} ${DaysTemp} )
+
+  for file in $( gawk '{print $11}' ${RUN_DIR}/new_liste_restart_files.txt ); do
     extract_date_file=$( echo ${file}  | sed -e "s/.*${JobName}_//" )
     date_file=$( echo ${extract_date_file} | sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
@@ -365 +321 @@
     if [ ${date_file} -lt ${date_end_pack} ] && [ ${date_file} -ge ${date_begin_pack} ] ; then 
         echo $( basename ${file} ) >> ${RUN_DIR}/liste_pack_${number_pack}.txt
+        echo $( ls -L ${file} ) >> ${RUN_DIR}/liste_remove_pack_${number_pack}.txt
+
 #    Will not work. Need to find a better ordering.
 #    elif [ ${date_file} -ge ${date_end_pack} ] ; then
@@ -371 +329 @@
   done
 
-  month_begin_pack=${month_end_pack}
-  year_begin_pack=${year_end_pack}
-  date_begin_pack=${year_begin_pack}$( printf "%02i\n" ${month_begin_pack} )01
-
-  # Here we create tar files directly in $STORE. Avoid to create then to copy.
-  tar -cvf ${STORE_RESTART}/${JobName}_${year_begin_pack}_${year_end_pack}_restart.tar --dereference --files-from ${RUN_DIR}/liste_pack_${number_pack}.txt
-  cat  ${RUN_DIR}/liste_pack_${number_pack}.txt | xargs rm 
+  # Create tar files
+  IGCM_debug_Print 1 "Tar ongoing"
+  tar -cvf ${JobName}_${date_begin_pack}_${date_end_pack}_restart.tar --dereference --files-from ${RUN_DIR}/liste_pack_${number_pack}.txt
+  # Save it
+  IGCM_sys_Put_Out ${JobName}_${date_begin_pack}_${date_end_pack}_restart.tar ${STORE_RESTART}
+  # Clean it
+  IGCM_sys_Rm ${JobName}_${date_begin_pack}_${date_end_pack}_restart.tar
+  # Clean links
+  cat ${RUN_DIR}/liste_pack_${number_pack}.txt | xargs rm
+  # Clean files links pointed to
+  cat ${RUN_DIR}/liste_remove_pack_${number_pack}.txt | xargs rm
+  # Save the list of files that has been pack (tar)
   mv ${RUN_DIR}/liste_pack_${number_pack}.txt ${STORE_RESTART}
+  IGCM_debug_Print 1 "Tar and cleaning done"
 
   (( number_pack = number_pack + 1 ))
 
+  # Add 1 day to date_end_pack to have the new date_begin_pack
+  date_begin_pack=$( $( IGCM_date_AddDaysToGregorianDate ${date_end_pack} 1 )
+
 done

trunk/libIGCM/libIGCM_post/libIGCM_post.ksh

@@ -486 +486 @@
     DaysTemp=$(( $( IGCM_date_DaysInPreviousPeriod ${PeriodDateEnd} ${config_Post_PackFrequency} end ) - 1 ))
     #
-    PackDateBegin=$( IGCM_date_AddDaysToGregorianDate ${PeriodDateEnd} -${DaysInPeriod} )
+    PackDateBegin=$( IGCM_date_AddDaysToGregorianDate ${PeriodDateEnd} -${DaysTemp} )
     #
     script=pack_debug
@@ -510 +510 @@
     IGCM_sys_MkdirWork ${POST_DIR}
     IGCM_debug_Verif_Exit_Post
-#    IGCM_sys_QsubPost ${script}
+    IGCM_sys_QsubPost ${script}
+EOF
+    #
+    script=pack_restart
+    #
+    Script_Post_Output=${script}.${PeriodDateEnd}
+    #
+    IGCM_sys_RshPost <<-EOF
+    export DEBUG_debug=${DEBUG_debug}
+    export MODIPSL=${MODIPSL}
+    export libIGCM_SX=${libIGCM}
+    export libIGCM=${libIGCM_POST}
+    export SUBMIT_DIR=${SUBMIT_DIR}
+    export MASTER=${MASTER}
+    export DateBegin=${PackDateBegin}
+    export DateEnd=${PeriodDateEnd}
+    export PeriodPack=${config_UserChoices_PackFrequency}
+    export StandAlone=false
+    export listVarEnv=${listVarEnv}
+    export Script_Post_Output=${Script_Post_Output}
+    . ${libIGCM_POST}/libIGCM_debug/libIGCM_debug.ksh
+    . ${libIGCM_POST}/libIGCM_sys/libIGCM_sys.ksh
+    IGCM_sys_MkdirWork ${POST_DIR}
+    IGCM_debug_Verif_Exit_Post
+    IGCM_sys_QsubPost ${script}
 EOF