Changeset 584

Timestamp:

03/19/12 10:36:39 (12 years ago)

Author:

sdipsl

Message:

• tune ncrcat command (exclude variables that are not in all files, instead of including those that are in every file because of command line lenght issues on titane)
• get the mesh_mask from the buffer if necessary
• correct some typo
• small modification due to the new .atlas_env_titane_ksh
• pack_output works in stand-alone ==> need to revisit dependancies with asynchronous rebuild and create_ts/create_se to have it online.

Location:

trunk/libIGCM

Files:

• AA_pack_output (modified) (3 diffs)
• libIGCM_post/IGCM_Patch_20091118_mask.ksh (modified) (1 diff)
• libIGCM_sys/libIGCM_sys_titane.ksh (modified) (1 diff)

trunk/libIGCM/AA_pack_output

@@ -272 +272 @@
 
   for comp in ${config_ListOfComponents[*]} ; do
-    dirList=$( find ${BUFR}/${comp}/ -maxdepth 1 -mindepth 1 -type d )
-    for dir in ${dirlist} ; do
+    dirList=$( find ${R_BUFR}/${comp}/Output -maxdepth 1 -mindepth 1 -type d )
+    for dir in ${dirList} ; do
       # dirID is like ATM.Output.MO
       dirID=$( echo $dir | sed "s:${R_BUFR}/::" | sed "s:/:.:g" )
@@ -284 +284 @@
       for myType in ${fileType} ; do
         grep ${myType} liste_files.${dirID}.txt > liste_files.${dirID}.${myType}.txt
+        nbfile=0
         for file in $( gawk '{print $11}' liste_files.${dirID}.${myType}.txt ); do
           extract_date_file=$( echo ${file}  | sed -e "s/.*${JobName}_[0-9]*_//" )
           date_file=$( echo ${extract_date_file} | sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
-          # echo pack n°${number_pack}  ${date_file} ${date_begin_pack} ${date_end_pack} 
+          # echo pack n°${number_pack}  ${date_file} ${date_begin_pack} ${date_end_pack}
           if [ ${date_file} -le ${date_end_pack} ] && [ ${date_file} -ge ${date_begin_pack} ] ; then 
             echo ${file} >> liste_pack_${myType}_${number_pack}.txt
+            ncdump -h ${file} | grep -E 'float|double' | cut -f 1 -d '(' | cut -f 2 -d ' ' >> liste_variables_${myType}_${number_pack}.txt
+            (( nbfile = nbfile + 1 ))
           fi
         done
+        # Select list of variables to work with
+        list_var=$( cat liste_variables_${myType}_${number_pack}.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d ',' )
+        liste_file_tmp=$( for i in $( cat liste_pack_${myType}_${number_pack}.txt ) ; do basename $i ; done )
         # Create packed files
         IGCM_debug_Print 1 "Ncrcat ongoing for ${dir} and ${myType}"
         output=${JobName}_${date_begin_pack}_${date_end_pack}_${myType}
-        cat liste_pack_${myType}_${number_pack}.txt | xargs IGCM_sys_ncrcat -o ${output}
+        #cat liste_pack_${myType}_${number_pack}.txt | xargs ncrcat -v ${list_var} -o ${output}
+        if [ X${list_var} = X ] ; then
+          ncrcat --omp_num_threads 4 -p ${dir} ${liste_file_tmp} --output ${output}
+        else
+          ncrcat --omp_num_threads 4 -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
+        fi
         # ------------------------------------------------------------------
         # Test if all was right before proceeding further
@@ -301 +312 @@
         IGCM_debug_Verif_Exit_Post
         # Save it
-        IGCM_sys_Put_Out ${output} ${R_SAVE}/${dir}
+        IGCM_sys_Put_Out ${output} ${R_SAVE}/$( echo $dir | sed "s:${R_BUFR}/::" )/${output}
         # ------------------------------------------------------------------
         # Test if all was right before proceeding further

trunk/libIGCM/libIGCM_post/IGCM_Patch_20091118_mask.ksh

@@ -20 +20 @@
     IGCM_debug_PushStack "IGCM_Patch_mask" $@
 
-    [ -f ${config_UserChoices_JobName}_mesh_mask.nc ] || IGCM_sys_Get ${R_SAVE}/OCE/Output/${config_UserChoices_JobName}_mesh_mask.nc .
+    if [ -f ${config_UserChoices_JobName}_mesh_mask.nc ] ; then
+      if [ -f ${R_SAVE}/OCE/Output/${config_UserChoices_JobName}_mesh_mask.nc ] ; then
+        IGCM_sys_Get ${R_SAVE}/OCE/Output/${config_UserChoices_JobName}_mesh_mask.nc .
+      elif [ -f ${R_BUFR}/OCE/Output/${config_UserChoices_JobName}_mesh_mask.nc ] ; then
+        IGCM_sys_Get ${R_BUFR}/OCE/Output/${config_UserChoices_JobName}_mesh_mask.nc .
+      fi
+    else
+      IGCM_debug_Exit "mesh_mask can not be found. Stop."
+      IGCM_debug_Exit "neither here ${R_SAVE}/OCE/Output/${config_UserChoices_JobName}_mesh_mask.nc"
+      IGCM_debug_Exit "nor here ${R_BUFR}/OCE/Output/${config_UserChoices_JobName}_mesh_mask.nc"
+      IGCM_debug_Verif_Exit_Post
+    fi
 
     filename=${1}

trunk/libIGCM/libIGCM_sys/libIGCM_sys_titane.ksh

@@ -92 +92 @@
 . /home/cont003/p86ipsl/.atlas_env_titane_ksh
 
-module switch nco/4.0.8 nco/3.9.4_netcdf4
+module switch nco/3.9.4 nco/3.9.4_netcdf4
 
 #====================================================