#!/bin/ksh #-Q- cesium #!/bin/ksh #-Q- cesium ###################### #-Q- cesium ## CESIUM CEA ## #-Q- cesium ###################### #-Q- cesium #MSUB -r TimeSeries # Nom du job #-Q- cesium #MSUB -N 1 # Reservation du noeud #-Q- cesium #MSUB -n 1 # Reservation du processus #-Q- cesium #MSUB -T 86400 # Limite de temps elapsed du job #-Q- cesium #MSUB -E "-j o" #-Q- cesium #MSUB -E "-S /bin/ksh" #-Q- platine #!/usr/bin/ksh #-Q- platine ################### #-Q- platine ## PLATINE CEA ## #-Q- platine ################### #-Q- platine #BSUB -J TimeSeries # Nom du job #-Q- platine #BSUB -N # message a la fin du job #-Q- platine #BSUB -n 1 # reservation des processeurs pour le job #-Q- platine #BSUB -W 1:00 # Limite temps #-Q- platine #BSUB -q post # Passage en queue post #-Q- sx8brodie #!/bin/ksh #-Q- sx8brodie ####################### #-Q- sx8brodie ## SX8BRODIE IDRIS ## #-Q- sx8brodie ####################### #-Q- sx8brodie # Temps Elapsed max. d'une requete hh:mm:ss #-Q- sx8brodie # @ wall_clock_limit = 20:00:00 #-Q- sx8brodie # Nom du travail LoadLeveler #-Q- sx8brodie # @ job_name = TimeSeries #-Q- sx8brodie # Fichier de sortie standard du travail #-Q- sx8brodie # @ output = $(job_name).$(jobid) #-Q- sx8brodie # Fichier de sortie d'erreur du travail #-Q- sx8brodie # @ error = $(job_name).$(jobid) #-Q- sx8brodie # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.) #-Q- sx8brodie # @ notification = error #-Q- sx8brodie # @ environment = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER #-Q- aix6 #!/bin/ksh #-Q- aix6 ####################### #-Q- aix6 ## VARGAS IDRIS ## #-Q- aix6 ####################### #-Q- aix6 # Temps Elapsed max. d'une requete hh:mm:ss #-Q- aix6 # @ wall_clock_limit = 20:00:00 #-Q- aix6 # Nom du travail LoadLeveler #-Q- aix6 # @ job_name = TimeSeries #-Q- aix6 # Fichier de sortie standard du travail #-Q- aix6 # @ output = $(job_name).$(jobid) #-Q- aix6 # Fichier de sortie d'erreur du travail #-Q- aix6 # @ error = $(job_name).$(jobid) #-Q- aix6 # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.) #-Q- aix6 # @ notification = error #-Q- aix6 # @ environment = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER #-Q- aix6 # @ queue #-Q- sx8brodie # @ queue #-Q- sx8mercure #!/bin/ksh #-Q- sx8mercure ###################### #-Q- sx8mercure ## SX8MERCURE CEA ## #-Q- sx8mercure ###################### #-Q- sx8mercure #PBS -N TimeSeries # Nom du job #-Q- sx8mercure #PBS -j o # regroupement des stdout et stderr #-Q- sx8mercure #PBS -S /usr/bin/ksh # shell de soumission #-Q- sx8mercure #PBS -l memsz_job=1gb # Limite memoire a 1 Go #-Q- sx8mercure #PBS -l cputim_job=24:00:00 # Limite temps a 2 heures #-Q- sx8mercure #PBS -q scalaire #-Q- sx9mercure #!/bin/ksh #-Q- sx9mercure ######################### #-Q- sx9mercure ## CESIUM FOR SX9 CEA ## #-Q- sx9mercure ######################### #-Q- sx9mercure #MSUB -r TimeSeries # Nom du job #-Q- sx9mercure #MSUB -N 1 # Reservation du noeud #-Q- sx9mercure #MSUB -n 1 # Reservation du processus #-Q- sx9mercure #MSUB -T 86400 # Limite de temps elapsed du job #-Q- sx9mercure #MSUB -E "-j o" #-Q- sx9mercure #MSUB -E "-S /bin/ksh" #-Q- titane #!/bin/ksh #-Q- titane ###################### #-Q- titane ## TITANE CEA ## #-Q- titane ###################### #-Q- titane #MSUB -r TimeSeries # Nom du job #-Q- titane #MSUB -N 1 # Reservation du noeud #-Q- titane #MSUB -n 1 # Reservation du processus #-Q- titane #MSUB -T 86400 # Limite de temps elapsed du job #-Q- titane #MSUB -E "-j o" #-Q- titane #MSUB -E "-S /bin/ksh" #-Q- titane ##MSUB -e nco.out # Sortie standard #-Q- titane ##MSUB -o nco.out # Sortie standard #-Q- lxiv8 ###################### #-Q- lxiv8 ## OBELIX LSCE ## #-Q- lxiv8 ###################### #-Q- lxiv8 #PBS -N TimeSeries #-Q- lxiv8 #PBS -m a #-Q- lxiv8 #PBS -j oe #-Q- lxiv8 #PBS -q medium #-Q- lxiv8 #PBS -o TimeSeries.$$ #-Q- lxiv8 #PBS -S /bin/ksh #-Q- default #!/bin/ksh #-Q- default ################## #-Q- default ## DEFAULT HOST ## #-Q- default ################## #************************************************************** # Author: Sebastien Denvil # Contact: Sebastien.Denvil@ipsl.jussieu.fr # $Date: $ # $Author: $ # $Revision: $ # IPSL (2006) # This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC # History: # Modification: # #************************************************************** # Check that everything went well during time series production # Display a short report # Launch what's missing # For use during a run (not on the end : PeriodState=Completed), it will complete # all TS to last PeriodDateEnd value, give by run.card->Configuration->OldPrefix string. # Chemin vers libIGCM libIGCM=${libIGCM:=::modipsl::/libIGCM} # Attention : à changer si la machine de post-traitement n'est pas la frontale du serveur de calcul ! # voir précence de la variable MirrorlibIGCM dans votre couche système. # Name Space of this experience SpaceName=${SpaceName:=PROD} # Expericence class of the run ExperimentName=${ExperimentName:=historical} # Name of this job JobName=${JobName:=v2.historical1} # répertoire courrant CURRENT_DIR=$( pwd ) # Emplacement des cartes CARD_DIR=${CARD_DIR:=${CURRENT_DIR}/${ExperimentName}/${JobName}} # répertoire de stockage des sorties des create_ts POST_DIR=${POST_DIR:=${CARD_DIR}/OutScript} if [ ! -d ${CARD_DIR} ]; then echo "No ${CARD_DIR}, we stop here" exit fi ######################################################################## . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh ; . ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh #; IGCM_debug_Check . ${libIGCM}/libIGCM_card/libIGCM_card.ksh #; IGCM_card_Check . ${libIGCM}/libIGCM_date/libIGCM_date.ksh #; IGCM_date_Check ######################################################################## #set -vx # First of all IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card UserChoices typeset option for option in ${config_UserChoices[*]} ; do IGCM_card_DefineVariableFromOption ${CARD_DIR}/config.card UserChoices ${option} done echo IGCM_debug_Print 1 "DefineVariableFromOption : config_UserChoices" IGCM_debug_PrintVariables 3 config_UserChoices_JobName #IGCM_debug_PrintVariables 3 config_UserChoices_SpaceName #IGCM_debug_PrintVariables 3 config_UserChoices_ExperimentName IGCM_debug_PrintVariables 3 config_UserChoices_CalendarType IGCM_debug_PrintVariables 3 config_UserChoices_DateBegin IGCM_debug_PrintVariables 3 config_UserChoices_DateEnd if [ -f ${CARD_DIR}/run.card ] ; then IGCM_card_DefineVariableFromOption ${CARD_DIR}/run.card Configuration OldPrefix IGCM_card_DefineVariableFromOption ${CARD_DIR}/run.card Configuration PeriodState IGCM_card_DefineVariableFromOption ${CARD_DIR}/run.card PostProcessing TimeSeriesCompleted IGCM_debug_Print 1 "DefineVariableFromOption : run_Configuration" IGCM_debug_PrintVariables 3 run_Configuration_OldPrefix IGCM_debug_PrintVariables 3 run_Configuration_PeriodState IGCM_debug_Print 1 "DefineVariableFromOption : run_PostProcessing" IGCM_debug_PrintVariables 3 run_PostProcessing_TimeSeriesCompleted if [ X${run_Configuration_PeriodState} != X"Completed" ] ; then DateEnd=$( IGCM_date_ConvertFormatToGregorian $( echo ${run_Configuration_OldPrefix} | awk -F'_' '{print $2}' ) ) else DateEnd=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} ) fi CompletedFlag=${run_PostProcessing_TimeSeriesCompleted} else DateEnd=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} ) CompletedFlag="" fi SavedCompletedFlag=${CompletedFlag} DateBegin=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} ) IGCM_date_GetYearMonth ${DateBegin} YearBegin MonthBegin IGCM_date_GetYearMonth ${DateEnd} YearEnd MonthEnd echo IGCM_debug_Print 1 "DateBegin for TimeSeries_Checker : "${DateBegin} IGCM_debug_Print 1 "DateEnd for TimeSeries_Checker : "${DateEnd} IGCM_debug_Print 1 "YearBegin MonthBegin for TimeSeries_Checker : ${YearBegin} ${MonthBegin}" IGCM_debug_Print 1 "YearEnd MonthEnd for TimeSeries_Checker : ${YearEnd} ${MonthEnd}" IGCM_debug_Print 1 "CompletedFlag = "${CompletedFlag} echo #==================================================== #R_SAVE : Job output directory if ( [ ! X${config_UserChoices_SpaceName} = X ] && [ ! X${config_UserChoices_ExperimentName} = X ] ) ; then FreeName=$( echo ${config_UserChoices_JobName} | sed 's/.*_//' ) R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName} R_DODS=${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName} else R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName} R_DODS=${config_UserChoices_TagName}/${config_UserChoices_JobName} fi IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents for comp in ${config_ListOfComponents[*]} ; do # Debug Print IGCM_debug_Print 1 ${comp} # Define component IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp} eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1 eval comptagname=\${config_ListOfComponents_${comp}[1]} > /dev/null 2>&1 # Read libIGCM compatibility version in ${compname}.card card=${CARD_DIR}/COMP/${compname}.card # Read and Build Output File stuff #IGCM_debug_Print 1 "DefineArrayFromOption : ${compname}_OutputFiles ${card}" IGCM_card_DefineArrayFromOption ${card} OutputFiles List ListFilesName=${compname}_OutputFiles_List eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1 # if [ X${FileName0} != X${NULL_STR} ] ; then # #IGCM_debug_Print 1 "Component : ${compname}" # # INITIALISATION # eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1 i=2 # until [ $i -ge $NbFiles ]; do # eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1 # if [ X${flag_post} != XNONE ] ; then # # First of all # IGCM_card_DefineArrayFromSection ${card} ${flag_post} # IGCM_card_DefineArrayFromOption ${card} ${flag_post} TimeSeriesVars IGCM_card_DefineArrayFromOption ${card} ${flag_post} Patches if [ X"$( eval echo \${${compname}_${flag_post}_TimeSeriesVars[*]} )" = X"Option not found ${flag_post}" ] ; then # New TimeSeriesVar description, with 2D, 3D and associate ChunckJob. unset ListDimension ListDimension[0]=2D ListDimension[1]=3D TimeSeries=false TimeSeries2D=false TimeSeries3D=false chunck=false iLoop=${#ListDimension[*]} j=0 until [ $j -ge ${iLoop} ]; do Dimension=${ListDimension[${j}]} IGCM_card_DefineArrayFromOption ${card} ${flag_post} TimeSeriesVars${Dimension} IGCM_card_DefineVariableFromOption ${card} ${flag_post} ChunckJob${Dimension} # # Time series WITHOUT chunk # if [ ! $( eval echo \${${compname}_${flag_post}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then if [ $( eval echo \${${compname}_${flag_post}_ChunckJob${Dimension}} ) = NONE ] ; then IGCM_debug_Print 2 "${Dimension} time series activated for ${flag_post}" eval TimeSeries${Dimension}=true chunck=false fi fi # # Time series WITH chunk # if [ ! $( eval echo \${${compname}_${flag_post}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then chunck_size=$( eval echo \${${compname}_${flag_post}_ChunckJob${Dimension}} ) if [ ! ${chunck_size} = NONE ] && [ ! ${chunck_size} = OFF ] ; then IGCM_debug_Print 2 "${Dimension} time series activated with ${chunck_size} chunck for ${flag_post}" eval TimeSeriesChunck${Dimension}=true chunck=true fi fi (( j=j+1 )) # # If TimeSeriesVars list is empty or OFF we skip # if ( [ $( eval echo \${${compname}_${flag_post}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] || \ [ ${chunck_size} = OFF ] ) ; then #IGCM_debug_Print 2 "Empty TS : ${compname}_Post_${FILE}_TimeSeriesVars${Dimension}" #(( i=i+3 )) continue fi # # ICI ON TESTE QUE LES FICHIERS TS SONT LA! # # FILE=$( echo ${flag_post} | awk "-FPost_" '{print $2}' ) IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension} IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension} # FlagDir=$( echo ${FILE} | awk -F "_" '{print $1}' ) case ${FlagDir} in *Y) TS_Dir=TS_YE ;; *M) TS_Dir=TS_MO ;; *D) TS_Dir=TS_DA ;; 3H|HF) TS_Dir=TS_HF ;; INS) TS_Dir=TS_INS ;; esac # # We need LIST of variables not allready produced (useful for standalone mode) # DateBegin=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} ) IGCM_date_GetYearMonth ${DateBegin} YearBegin MonthBegin # Si on n'a pas de chunck, dans ce cas chunck_size=la durée de la simulation en années YearsChunckLength=$( echo ${chunck_size} | sed -e "s/[yY]//" ) [ ${chunck} = false ] && YearsChunckLength=$(( YearEnd - YearBegin + 1 )) NbYearsChunckLoop=$(( ( YearEnd - YearBegin + 1 ) / YearsChunckLength )) Reste=$(( ( YearEnd - YearBegin + 1 ) % YearsChunckLength )) if [ ${Reste} -ne 0 ] ; then NbYearsChunckLoop=$(( NbYearsChunckLoop + 1 )) fi if [ ${NbYearsChunckLoop} -eq 1 ] ; then PeriodDateEnd=${DateEnd} else DaysInYear=$( IGCM_date_DaysInYear ${YearBegin} ) PeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( YearsChunckLength * DaysInYear - 1 )) ) fi # DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir} YearsChunckLoop=1 ChunckDebut=${DateBegin} ChunckFin=${PeriodDateEnd} while [ ${YearsChunckLoop} -le ${NbYearsChunckLoop} ] ; do countTotal=0 countGood=0 countBad=0 for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do TestedFile=${config_UserChoices_JobName}_${ChunckDebut}_${ChunckFin}_${FlagDir}_${var}.nc # if [ ! -f ${DIRECTORY}/${TestedFile} ] ; then (( countBad = countBad + 1 )) [ ${countBad} = 1 ] && IGCM_debug_Print 3 "Missing time series from ${FILE} :" IGCM_debug_Print 3 ${DIRECTORY}/${TestedFile} else (( countGood = countGood + 1 )) fi (( countTotal = countTotal + 1 )) done SuccessRate=$(( countGood * 100 / countTotal )) if [ ${SuccessRate} -ne 100 ] ; then IGCM_debug_Print 2 -e "\033[1;31m${SuccessRate}% files OK.\033[m for period ${ChunckDebut}-${ChunckFin}" else IGCM_debug_Print 2 -e "\033[1;32m${SuccessRate}% files OK.\033[m for period ${ChunckDebut}-${ChunckFin}" fi echo if ( [ ${chunck} = true ] && [ ${SuccessRate} -ne 100 ] ) ; then ChunckCompletedFlag="" CompletedFlag=${SavedCompletedFlag} if [ X${CompletedFlag} != X ] ; then ( [ ${CompletedFlag} -gt ${ChunckDebut} ] && [ ${CompletedFlag} -le ${ChunckFin} ] ) && ChunckCompletedFlag=${CompletedFlag} fi IGCM_debug_Print 2 -e "\033[1;31mSubmit ${FILE} chunck ${Dimension}\033[m period ${ChunckDebut}-${ChunckFin}" listVarEnv="libIGCM,SUBMIT_DIR,POST_DIR,DateBegin,PeriodDateEnd,CompletedFlag,TsTask,CompToRead,FlagToRead" export libIGCM=${libIGCM} export SUBMIT_DIR=${CARD_DIR} export POST_DIR=${POST_DIR} export DateBegin=${ChunckDebut} export PeriodDateEnd=${ChunckFin} export CompletedFlag=${ChunckCompletedFlag} export TsTask=Chunck${Dimension} export CompToRead=${comp} export FlagToRead=${i} export listVarEnv=${listVarEnv} IGCM_sys_MkdirWork ${POST_DIR} IGCM_debug_Verif_Exit IGCM_sys_QsubPost create_ts echo fi if ( [ ${chunck} = false ] && [ ${SuccessRate} -ne 100 ] ) ; then eval Launch${Dimension}=true fi # Date update ChunckDebut=$( IGCM_date_AddDaysToGregorianDate ${ChunckFin} 1 ) (( YearsChunckLoop = YearsChunckLoop + 1 )) if [ ${YearsChunckLoop} -eq ${NbYearsChunckLoop} ] ; then ChunckFin=${DateEnd} else #PeriodDateEnd=$(( YearBegin + YearsChunckLength - 1 )) ChunckFin=$( IGCM_date_AddDaysToGregorianDate ${ChunckDebut} $(( YearsChunckLength * DaysInYear - 1 )) ) fi done done else ListDimension[0]="" TimeSeries=true TimeSeries2D=false TimeSeries3D=false TimeSeriesChunck2D=false TimeSeriesChunck3D=false fi fi (( i=i+3 )) done fi done # comp loop echo DateBegin=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} ) IGCM_date_GetYearMonth ${DateBegin} YearBegin MonthBegin CompletedFlag=${SavedCompletedFlag} if [ X${Launch2D} = Xtrue ] ; then IGCM_debug_Print 2 -e "\033[1;31mSubmit 2D\033[m without chunck period ${DateBegin}-${DateEnd}" listVarEnv="libIGCM,SUBMIT_DIR,POST_DIR,DateBegin,PeriodDateEnd,CompletedFlag,TsTask" export libIGCM=${libIGCM} export SUBMIT_DIR=${CARD_DIR} export POST_DIR=${POST_DIR} export DateBegin=${DateBegin} export PeriodDateEnd=${DateEnd} export CompletedFlag=${CompletedFlag} export TsTask=2D export listVarEnv=${listVarEnv} IGCM_sys_MkdirWork ${POST_DIR} IGCM_debug_Verif_Exit IGCM_sys_QsubPost create_ts echo fi if [ X${Launch3D} = Xtrue ] ; then IGCM_debug_Print 2 -e "\033[1;31mSubmit 3D\033[m without chunck period ${DateBegin}-${DateEnd}" listVarEnv="libIGCM,SUBMIT_DIR,POST_DIR,DateBegin,PeriodDateEnd,CompletedFlag,TsTask" export libIGCM=${libIGCM} export SUBMIT_DIR=${CARD_DIR} export POST_DIR=${POST_DIR} export DateBegin=${DateBegin} export PeriodDateEnd=${DateEnd} export CompletedFlag=${CompletedFlag} export TsTask=3D export listVarEnv=${listVarEnv} IGCM_sys_MkdirWork ${POST_DIR} IGCM_debug_Verif_Exit IGCM_sys_QsubPost create_ts echo fi