#-Q- curie ###################### #-Q- curie ## CURIE TGCC/CEA ## #-Q- curie ###################### #-Q- curie #MSUB -r TS # Nom du job #-Q- curie #MSUB -o TS.out_%I #-Q- curie #MSUB -e TS.out_%I #-Q- curie #MSUB -n 1 # Reservation du processus #-Q- curie #MSUB -T 80000 # Limite de temps elapsed du job #-Q- curie #MSUB -q ::default_node:: #-Q- curie #MSUB -c ::default_core:: #-Q- curie #MSUB -Q normal #-Q- curie #MSUB -A ::default_project:: #-Q- curie set +x #-Q- irene ###################### #-Q- irene ## IRENE TGCC/CEA ## #-Q- irene ###################### #-Q- irene #MSUB -r TS # Job name #-Q- irene #MSUB -o TS.out_%I #-Q- irene #MSUB -e TS.out_%I #-Q- irene #MSUB -n 1 # Number of cores #-Q- irene #MSUB -T 80000 # Maximum elapsed time #-Q- irene #MSUB -q ::default_node:: #-Q- irene #MSUB -c ::default_core:: #-Q- irene #MSUB -Q normal #-Q- irene #MSUB -A ::default_post_project:: #-Q- irene #MSUB -m store,work,scratch #-Q- irene set +x #-Q- irene-amd ###################### #-Q- irene-amd ## IRENE TGCC/CEA ## #-Q- irene-amd ###################### #-Q- irene-amd #MSUB -r TS # Job name #-Q- irene-amd #MSUB -o TS.out_%I #-Q- irene-amd #MSUB -e TS.out_%I #-Q- irene-amd #MSUB -n 1 # Number of cores #-Q- irene-amd #MSUB -T 80000 # Maximum elapsed time #-Q- irene-amd #MSUB -q ::default_node:: #-Q- irene-amd #MSUB -c ::default_core:: #-Q- irene-amd #MSUB -Q normal #-Q- irene-amd #MSUB -A ::default_post_project:: #-Q- irene-amd #MSUB -m store,work,scratch #-Q- irene-amd set +x #-Q- jeanzay #!/bin/ksh #-Q- jeanzay ###################### #-Q- jeanzay ## JEANZAY IDRIS ## #-Q- jeanzay ###################### #-Q- jeanzay #SBATCH --job-name=TS # Job Name #-Q- jeanzay #SBATCH --output=TS.out_%I # standard output #-Q- jeanzay #SBATCH --error=TS.out_%I # error output #-Q- jeanzay #SBATCH -N 1 # Number of core #-Q- jeanzay #SBATCH --partition=prepost # Post-processing partition #-Q- jeanzay #SBATCH --time=10:00:00 # Wall clock limit (seconds) #-Q- jeanzay #SBATCH --account ::default_project::@cpu #-Q- jeanzay set +x #-Q- ada #!/bin/ksh #-Q- ada ####################### #-Q- ada ## ADA IDRIS ## #-Q- ada ####################### #-Q- ada # @ job_type = mpich #-Q- ada # @ requirements = (Feature == "prepost") #-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss #-Q- ada # @ wall_clock_limit = 20:00:00 #-Q- ada # Nom du travail LoadLeveler #-Q- ada # @ job_name = TS #-Q- ada # Fichier de sortie standard du travail #-Q- ada # @ output = $(job_name).$(jobid) #-Q- ada # Fichier de sortie d'erreur du travail #-Q- ada # @ error = $(job_name).$(jobid) #-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.) #-Q- ada # @ notification = error #-Q- ada # @ environment = $DEBUG_debug ; $BigBrother ; $postProcessingStopLevel ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $POST_DIR ; $Script_Post_Output ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateEnd ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; wall_clock_limit=$(wall_clock_limit) #-Q- ada # @ queue #-Q- obelix ###################### #-Q- obelix ## OBELIX LSCE ## #-Q- obelix ###################### #-Q- obelix #PBS -N TS #-Q- obelix #PBS -m a #-Q- obelix #PBS -j oe #-Q- obelix #PBS -q medium #-Q- obelix #PBS -o TS.$$ #-Q- obelix #PBS -S /bin/ksh #-Q- mesoipsl #!/bin/ksh #-Q- mesoipsl ###################### #-Q- mesoipsl ## MESO ESPRI IPSL ## #-Q- mesoipsl ###################### #-Q- mesoipsl #SBATCH --job-name=TS # Job Name #-Q- mesoipsl #SBATCH --output=TS.out_%I # standard output #-Q- mesoipsl #SBATCH --error=TS.out_%I # error output #-Q- mesoipsl #SBATCH -N 1 # Number of core #-Q- mesoipsl #SBATCH --time=10:00:00 # Wall clock limit (seconds) #-Q- ifort_CICLAD ###################### #-Q- ifort_CICLAD ## CICLAD IPSL ## #-Q- ifort_CICLAD ###################### #-Q- ifort_CICLAD #PBS -N TS #-Q- ifort_CICLAD #PBS -m a #-Q- ifort_CICLAD #PBS -j oe #-Q- ifort_CICLAD ###PBS -q h12 # Queue for 12 hours at ciclad only #-Q- ifort_CICLAD #PBS -S /bin/ksh #-Q- default #!/bin/ksh #-Q- default ################## #-Q- default ## DEFAULT HOST ## #-Q- default ################## #************************************************************** # Author: Sebastien Denvil # Contact: Sebastien.Denvil__at__ipsl.jussieu.fr # $Revision:: $ Revision of last commit # $Author:: $ Author of last commit # $Date:: $ Date of last commit # IPSL (2006) # This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC # #************************************************************** #set -eu #set -vx date #-Q- ada export OMP_NUM_THREADS=1 #D- Task type DO NOT CHANGE (computing, post-processing or checking) TaskType=post-processing ######################################################################## #D- Flag to determine if this job in a standalone mode #D- Default : value from AA_job if any StandAlone=${StandAlone:=true} #D- Path to libIGCM #D- Default : value from AA_job if any libIGCM=${libIGCM:=::modipsl::/libIGCM} #D- Flag to determine allready produced time series. Empty if you start from the beginning #D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series CompletedFlag=${CompletedFlag:=} #D- Flag to determine task type this script will perform. #D- Possible Value : 2D, 3D, Chunck2D, Chunck3D #D- Default : Value from AA_job if any. Usefull in StandAlone case. TsTask=${TsTask:=2D} if [ X${TsTask} = Xempty ] ; then TsTask="" fi #D- Flag to determine if rebuild process has been performed asynchronously #D- Possible value true or false. #D- If true raw files has allready been patched by rebuild job #D- If false create_ts will patch the raw files #D- Default : Value from AA_job if any. Usefull in StandAlone case. RebuildFrequency=${RebuildFrequency:=true} #D- Flag to determine job's output directory #D- Default : value from libIGCM_post.ksh if any POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}} #D- Increased verbosity (1, 2, 3) #D- Default : value from AA_job if any Verbosity=${Verbosity:=3} #D- Low level debug : to bypass lib test checks and stack construction #D- Default : value from AA_job if any DEBUG_debug=${DEBUG_debug:=false} #D- TEMPORARY Flag to determine atmospheric resolution #D- Default : value from atmospheric driver if any RESOL_ATM=${RESOL_ATM:=ALL} #D- Flag to determine surface resolution #D- Default : value from surface driver if any RESOL_SRF=${RESOL_SRF:=ALL} #D- Flag to determine surface resolution #D- Default : value from surface driver if any RESOL_SBG=${RESOL_SBG:=ALL} #D- TEMPORARY Flag to determine ocean resolution #D- Default : value from ocean driver if any RESOL_OCE=${RESOL_OCE:=ORCA2} #D- TEMPORARY Flag to determine ice resolution #D- Default : value from ice driver if any RESOL_ICE=${RESOL_OCE:=ORCA2} #D- TEMPORARY Flag to determine marine biogeochemistry resolution #D- Default : value from ice driver if any RESOL_MBG=${RESOL_OCE:=ORCA2} ######################################################################## . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh . ${libIGCM}/libIGCM_card/libIGCM_card.ksh . ${libIGCM}/libIGCM_date/libIGCM_date.ksh #------- . ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh . ${libIGCM}/libIGCM_config/libIGCM_config.ksh #------- IGCM_sys_MkdirWork ${RUN_DIR_PATH} IGCM_sys_Cd ${RUN_DIR_PATH} #------- ( ${DEBUG_debug} ) && IGCM_debug_Check ( ${DEBUG_debug} ) && IGCM_card_Check ( ${DEBUG_debug} ) && IGCM_date_Check ######################################################################## #set -vx # Useful cleaning function MENAGE () { typeset ExistFlag ExistFlag=$( IGCM_sys_RshArchive "[ -f $1/$3 ] && echo 0 || echo 1" ) [ ${ExistFlag} -eq 0 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;) } ######################################################################## # ------------------------------------------------------------------ # Test if all was right before proceeding further # ------------------------------------------------------------------ IGCM_debug_Verif_Exit if [ ${StandAlone} = true ] ; then CARD_DIR=${SUBMIT_DIR} else CARD_DIR=${RUN_DIR_PATH} IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH} IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH} IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH} IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH} fi #================================== # First of all # # Read libIGCM compatibility version in config.card # Read UserChoices section # Read Ensemble section # Read Post section # Define all netcdf output directories #================================== IGCM_config_CommonConfiguration ${CARD_DIR}/config.card # ------------------------------------------------------------------ # Activate BigBrother so as to supervise this job # ------------------------------------------------------------------ IGCM_debug_BigBro_Initialize #================================== # Define default value to keep compatibility with previous card: means before changes due to TGCC if [ X${PackDefault} = Xtrue ] ; then [ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency} else config_Post_PackFrequency=NONE fi #================================== # If pack is active then PackFrequency overule the config_UserChoices_PeriodLength if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then config_UserChoices_PeriodLength=${config_Post_PackFrequency} fi # # Determine component to take care of depending on kind of task create_ts will perform # [ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} || IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents # # Determine period for time series. Default : value from AA_job or from launch_TS if any # [ X${CompletedFlag} = X ] || CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} ) DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )} # PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )} # # Determine Dimension of the produced time series : 2D or 3D if ( [ X${TsTask} = XChunck2D ] || [ X${TsTask} = X2D ] ) ; then Dimension=2D elif ( [ X${TsTask} = XChunck3D ] || [ X${TsTask} = X3D ] ) ; then Dimension=3D else Dimension="" fi # ------------------------------------------------------------------ #D- Test if all was right before entering the loop # ------------------------------------------------------------------ IGCM_debug_Verif_Exit ######################################################################## # Depending on requested time series fill following variables : # # - LISTE_FILE_${comp}[*] # - LISTE_VARS_${file}[*] # - LISTE_AXIS_${file}[*] # - LISTE_PATCH_${file}[*] ######################################################################## # # For each selected component determine which files need post-processing # DoJob=false for comp in ${config_ListOfComponents[*]} ; do # ActiveComp=false # IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp} eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1 # card=${CARD_DIR}/COMP/${compname}.card IGCM_card_DefineArrayFromOption ${card} OutputFiles List # ListFilesName=${compname}_OutputFiles_List eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1 # if [ X${FileName0} != X${NULL_STR} ] ; then # #IGCM_debug_Print 1 "Component : ${compname}" # # INITIALISATION # typeset i if [ ${FlagToRead} ] ; then ((NbFiles=${FlagToRead})) ((i=${NbFiles})) else eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1 i=2 fi # until [ $i -gt $NbFiles ]; do # eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1 # if [ X${flag_post} != XNONE ] ; then # # For each comp and each file who needs post-processing # determine which variables need time series. # FILE=$( echo ${flag_post} | awk "-FPost_" '{print $2}' ) IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension} IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension} # # If TimeSeriesVars list is empty we skip # if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then # IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}" IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty" (( i=i+3 )) continue # fi # # If files/variables has been switch off by ChunckJob${Dimension}=0 we skip # if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}" IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off" (( i=i+3 )) continue fi # # If we are not a Chunck type task, we skip files/variables handled by chunck task # if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}" IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )" (( i=i+3 )) continue fi fi # # Now on we know we have things to do # # # We need coordinates axis to properly fill produced time series # IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal # if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then # IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}" IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!" (( i=i+3 )) continue # fi # # We need LIST of variables not allready produced (useful for standalone mode) # FlagDir=$( echo ${FILE} | awk -F "_" '{print $1}' ) case ${FlagDir} in *Y) TS_Dir=TS_YE ;; *M) TS_Dir=TS_MO ;; *D) TS_Dir=TS_DA ;; HF) TS_Dir=TS_HF ;; 3H) TS_Dir=TS_HF ;; INS) TS_Dir=TS_INS ;; esac # count=0 for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do # DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir} TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc # IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile} if [ ! $? = 0 ] ; then eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var} ActiveComp=true (( count = count + 1 )) fi done # if [ ${count} -gt 0 ] ; then # eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE} eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]} # if [ ! X${RebuildFrequency} = Xtrue ] ; then IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]} if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do if [ Xload${Patch} != X ] ; then . ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh eval load${Patch}=loaded fi done fi fi fi fi (( i=i+3 )) done fi # if [ ${ActiveComp} = true ] ; then set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp} fi done set -A config_ListOfComponents ${ActiveListOfComponents[*]} [ ! -z "${ActiveListOfComponents}" ] && DoJob=true ######################################################################## # # IS THERE SOME ALLREADY PRODUCED TIME SERIES ? # IF SO BRING THEM IN THE WORKING DIRECTORY # # PS : Keep in mind that IGCM_sys_Get here is a weak link : # - especially for IDRiS # - no special protection against "command too long" ! # - libIGCM_sys should handle that or modify following code part # ######################################################################## if [ ${StandAlone} != true ] ; then Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \ ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \ echo \${run_PostProcessing_TimeSeriesRunning}" ) #if [ X${Running_Flag} = Xy ] ; then # # Time Series Job allready running # IGCM_debug_Print 1 "Time Series Job allready running exit" # exit #fi # Now run.card know we are running IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y" # Get information from last execution #CompletedFlag=$( IGCM_sys_RshMaster \ # "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\ # echo \${run_PostProcessing_TimeSeriesCompleted}" ) fi # Is it the first submission or not ? if ( [ X${CompletedFlag} = X ] || [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then # First Time Series Submission FIRST_PASS=TRUE Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin) DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) ) else # Last Completed Time Series FIRST_PASS=FALSE DATE_FIN_JOB_B=${CompletedFlag} date=${DateBegin}_${DATE_FIN_JOB_B} # for comp in ${config_ListOfComponents[*]} ; do # IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp} # IGCM_sys_Cd ${RUN_DIR_PATH}/${comp} # unset liste_file # file_path=${R_SAVE}/${comp}/Analyse # i=0 file=${NULL_STR} # for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do # # Determine in which output can we find file # IE : Analyse/TS or Analyse/DA2TS or ... # FlagDir=$( echo ${file} | awk -F "_" '{print $1}' ) case ${FlagDir} in *Y) TS_Dir=TS_YE ;; *M) TS_Dir=TS_MO ;; *D) TS_Dir=TS_DA ;; HF) TS_Dir=TS_HF ;; 3H) TS_Dir=TS_HF ;; INS) TS_Dir=TS_INS ;; esac for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do # IGCM_sys_TestFileArchive ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc if [ $? = 0 ] ; then liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc else IGCM_debug_Print 1 "WARNING ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc dont exist" fi # done done if [ X${file} != X${NULL_STR} ] ; then IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp} IGCM_debug_Verif_Exit fi done # LEVEL=0 # fi ######################################################################## # # DEFINE LOOP PARAMETERS # ######################################################################## DATE_COUNT=${DATE_FIN_JOB_B} DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B} ( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 || NBRE_FILE_TOT=1 if [ ${DoJob} = true ] ; then while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do (( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 )) Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} ) DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} ) done else NBRE_FILE_TOT=0 fi if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then NBRE_TOUR=1 elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT} else let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1 fi CURRENT_LOOP=1 # THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN. # WE HAVE 3 LEVELS # - LEVEL 1 : JUST ONE LOOP TO COMPLETE # - LEVEL 2 : WE ARE IN THE "DEEP" LOOP # - LEVEL 3 : WE ARE IN THE LAST LOOP while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then # - LEVEL 1 : JUST ONE LOOP TO COMPLETE NBRE_FILE_LOOP=${NBRE_FILE_TOT} if [ ${FIRST_PASS} = TRUE ] ; then DATE_COURANTE=${DATE_FIN_JOB_B_LOOP} else Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} ) DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} ) fi DATE_FIN=${PeriodDateEnd} DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP} [ -n "${LEVEL}" ] && FLAG_B="TRUE" || FLAG_B="FALSE" LEVEL=1 DEPOT="TRUE" elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then # - LEVEL 2 : WE ARE IN THE "DEEP" LOOP NBRE_FILE_LOOP=${UNIX_MAX_LIMIT} if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} ) DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} ) elif [ -z "${DATE_COURANTE}" ] ; then DATE_COURANTE=${DATE_FIN_JOB_B} else Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} ) DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} ) fi (( TotLength = 0 )) COMPTEUR=1 DATE_LOOP=${DATE_COURANTE} while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do # Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} ) DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} ) (( TotLength = TotLength + Length )) (( COMPTEUR = COMPTEUR + 1 )) # done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} ) [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP} [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" || [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" || FLAG_B="FALSE" DEPOT="FALSE" LEVEL=2 else # - LEVEL 3 : WE ARE IN THE LAST LOOP NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) )) [ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT} Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} ) DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} ) DATE_FIN=${PeriodDateEnd} [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP} LEVEL=3 DEPOT="TRUE" FLAG_B="TRUE" fi # THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN. for comp in ${config_ListOfComponents[*]} ; do # IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp} # IGCM_sys_Cd ${RUN_DIR_PATH}/${comp} # eval R_OUT_${comp}=${R_SAVE}/${comp} # i=0 # for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do # # Determine in which output we can find file # IE : Output/MO or Output/DA or ... # FlagDir=$( echo ${file} | awk -F "_" '{print $1}' ) case ${FlagDir} in *Y) TS_Dir=TS_YE ; FreqDir=YE ;; *M) TS_Dir=TS_MO ; FreqDir=MO ;; *D) TS_Dir=TS_DA ; FreqDir=DA ;; HF) TS_Dir=TS_HF ; FreqDir=HF ;; 3H) TS_Dir=TS_HF ; FreqDir=HF ;; INS) TS_Dir=TS_INS ; FreqDir=INS ;; *) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}." IGCM_debug_Exit "Job create_ts" ;; esac # Initialize array unset liste_file unset liste_file_tmp eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/ COMPTEUR=1 MissingFile=FALSE DATE_LOOP=${DATE_COURANTE} while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do # Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end) DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP} Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} ) DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} ) IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc if [ $? = 0 ] ; then liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc else IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist" MissingFile=TRUE fi (( COMPTEUR = COMPTEUR + 1 )) # done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done # We skip this file type if one of them is missing. if [ "${MissingFile}" = "TRUE" ] ; then IGCM_debug_Print 1 "WARNING We skip ${file} type processing" continue fi DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} ) # Get selected files IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp} IGCM_debug_Verif_Exit # Apply IGCM_Patch if needed if [ ! X${RebuildFrequency} = Xtrue ] ; then if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then for file_t in $( eval echo ${liste_file_tmp[*]} ); do for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do IGCM_${Patch} ${file_t} IGCM_debug_Verif_Exit done done fi fi # WE CAN CONCATENATE FILES AT THIS POINT [ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC} Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end) DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) ) DATE_BUILD=${DATE_BUILD1}_${DATE_FIN} DATE_BUILD_END=${DateBegin}_${DATE_FIN} liste_coord=" " for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do liste_coord=${liste_coord}${axis}"," done # Time axis must be the last dimension time_axis=${axis} # SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do # # We skip variables not in the first file of the list. # Seems overkill to test thel all... but will do if needed ncdump -hv ${var} ${liste_file_tmp[0]} > /dev/null 2>&1 if [ ! $? = 0 ] ; then IGCM_debug_Print 1 "WARNING We skip ${var}, not present within ${liste_file_tmp[0]}" continue fi # if [ "${FLAG_B}" = "TRUE" ] ; then # WE CONCATENATE WITH EXISTING FILES file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out} IGCM_debug_Verif_Exit # IGCM_sys_Rm ${file1} if [ ! "${DEPOT}" = "TRUE" ] ; then eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out} # "${file1} = file_before ? [ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out} fi else # OR NOT file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${file_out} IGCM_debug_Verif_Exit fi # if [ "${DEPOT}" = "TRUE" ] ; then # WE PUT FINAL FILE ON FILE SYSTEM [ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc # eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out} [ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out} if [ ! "X${file1}" = "X" ] ; then [ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out} fi # [ -f ${file_out} ] && IGCM_sys_Rm ${file_out} # fi # done # for var in ${liste_var} ; do IGCM_sys_Rm ${liste_file_tmp[*]} done # for file in ${liste_file_brut} do done # for comp in ${config_ListOfComponents[*]} ; do # GO AHEAD IN DEEPEST LOOP (( CURRENT_LOOP = CURRENT_LOOP + 1 )) done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do # FINISH # DODS copy for comp in ${config_ListOfComponents[*]} ; do for TS_Dir in TS_YE TS_MO TS_DA TS_HF TS_INS ; do IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir} [ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir} done done export DEBUG_debug; export BigBrother ; export libIGCM; export SUBMIT_DIR; export POST_DIR; export Script_Output ; export StandAlone ; export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ; export RESOL_SBG listVarEnv="DEBUG_debug,BigBrother,libIGCM,SUBMIT_DIR,POST_DIR,Script_Post_Output,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG" export listVarEnv #D- # -------------------------------------------------------------------- #D- Test if all was right # -------------------------------------------------------------------- IGCM_debug_Verif_Exit # -------------------------------------------------------------------- #D- Submit MONITORING # -------------------------------------------------------------------- if ( [ X${TsTask} = X2D ] || [ X${TsTask} = X3D ] || [ X${TsTask} = X ] ) ; then # From 2D and 3D it is the last one to finish that will submit the monitoring. if [ ! "X$( grep "IGCM_sys_RmRunDir" ${POST_DIR}/create_ts.${PeriodDateEnd}.[23]D.out 2> /dev/null )" = "X" ] ; then Script_Post_Output=monitoring.${PeriodDateEnd} IGCM_sys_QsubPost monitoring fi fi #D- # -------------------------------------------------------------------- #D- Update the run.card # -------------------------------------------------------------------- if [ ${StandAlone} != true ] ; then # Put in run.card end period of time series [ X"${DATE_FIN}" = X ] || IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}" # We have finish the job ! IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n" fi # Clean RUN_DIR_PATH (necessary for cesium and titane only) IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH} # ------------------------------------------------------------------ # Finalize BigBrother to inform that the jobs end # ------------------------------------------------------------------ IGCM_debug_BigBro_Finalize date