source: TOOLS/SURPACK_IPSL/create_config_card.sh @ 3740

#!/bin/bash
##
## Job permettant de recreer une partie du config.card d'une simulation
## Champs vises : JobName / DateBegin / DateEnd / Calendar 
##

# Prend en argument un fichier contenant une liste de simulations
# cree pour chacune son config.card
# Ex : ./create_config_card.job liste_simul_fichier_param_test1.txt liste_fichier_dmnfs
# Ex : ./create_config_card.job liste_simul_param_SL.txt /cscratch/cont003/labetoul/FROM_CCRT_INFORMATIONS_DMNFS/information_dmnfs_2012-03-21


LISTE_SIMUL=${1}
LISTE_DMNFS=${2}

# ----
export JOB_DIR=${JOB_DIR:-${TMP_MIGR_DATA}}
export IGCM_DEM=${IGCM_DEM:-${SCRATCHDIR}/IGCM_DEM}
export IGCM_TMP=${IGCM_TMP:-${IGCM_DEM}/tmp}
mkdir -p ${IGCM_DEM}
mkdir -p ${IGCM_TMP}
clean_tmp=${clean_tmp:-yes}
EXE_DIR=$(dirname ${0})
SCRIPT_NAME=$(basename ${0} )
source ${EXE_DIR}/DEM_utilities.sh

# gpdebug : verif si le present script a ete execute jusqu'au bout #########
MonitoringInfo="${SCRIPT_NAME}-->OK"
generalMonitoring "$MonitoringInfo"

DEM_log -0 "Demarrage de ${SCRIPT_NAME}"

# gpdebug : on augmente la liste des proc à détruire en cas d'erreur bloquante du présent proc
export listPID="$$ ${listPID}" #########

cd ${JOB_DIR}
# ----

DEM_log -0 "LISTE_SIMUL : ${LISTE_SIMUL}"
DEM_log -0 "LISTE_DMNFS : ${LISTE_DMNFS}"

[[ -f ${IGCM_DEM}/config_card.liste ]] && rm ${IGCM_DEM}/config_card.liste


for PATH_SIMUL_FULL in `awk  '{print $0}' ${LISTE_SIMUL}` ; do 

    DEM_log -3 PATH_SIMUL_FULL : ${PATH_SIMUL_FULL}
    PATH_SIMUL_SHORT=$(echo ${PATH_SIMUL_FULL} | sed "s%${INPUT_DMF_DATA}/%%" ) # gpdebug : added
    DEM_log -3 PATH_SIMUL_SHORT : ${PATH_SIMUL_SHORT}

    IGCM_DEM_SIMU=${IGCM_DEM}/${PATH_SIMUL_SHORT}
    mkdir -p ${IGCM_DEM_SIMU}
    rm -rf ${IGCM_DEM_SIMU}/*

    STORE_CP=${IGCM_DEM_SIMU}/store_cp
    mkdir -p ${STORE_CP}

### note dans cette version on ne gere pas les fichiers Monitoring ils sont deja la ou il faut et comme il faut
    # on recherche le nom du job
    JobName=""
    

    # Etablir la liste des fichiers d'Analyse
    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Analyse  | awk '{print $3}' |  sort >> ${IGCM_TMP}/${$}.cp_analyse_files.list
    awk -F/ '{if ($(NF)=="Analyse") print $0} ' ${IGCM_TMP}/${$}.cp_analyse_files.list > ${STORE_CP}/cp_files.list
    rm -f ${IGCM_TMP}/${$}.cp_analyse_files.list


    # Etablir la liste des fichiers de restart
    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep RESTART | awk '{print $2 " " $3}' | grep -e '.*/RESTART/.*\..*' |grep -v "jnl" | sort  >> ${IGCM_TMP}/${$}.liste_restart_files_config.txt

    # Retirons de la liste les fichiers que l'on n'attends pas, en particulier si il  y a des sous-repertoires au repetoire RESTART/
    for file in `awk  '{print $2}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt ` ; do 
        dir=$( echo $file | sed -e "s%.*RESTART/%%") 
        testdir=$(dirname ${dir}) 
        if [ $testdir != "." ]; then 
            echo $( grep ${file} ${IGCM_TMP}/${$}.liste_restart_files_config.txt)  >> ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt
        fi
    done
    if [ -f ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt ]; then 
        join -t % -v 1 ${IGCM_TMP}/${$}.liste_restart_files_config.txt ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt
        mv ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.liste_restart_files_config.txt
    fi
    
    # Etablir la liste des fichiers output
    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Output  | awk '{print $2 " "  $3}' | grep -e '.*/Output/.*\.nc$'  | sort >> ${IGCM_TMP}/${$}.liste_output_files_config.txt
    # Retirons de la liste les fichiers que l'on n'attends pas, en particulier si il  y a des sous-repertoires aux repetoires Output/*/
    for file in `awk  '{print $2}'  ${IGCM_TMP}/${$}.liste_output_files_config.txt ` ; do 
        dir=$( echo $file | sed -e "s%.*Output/[^/]*/%%") 
        testdir=$(dirname ${dir}) 
        if [ $testdir != "." ]; then 
            echo $( grep ${file} ${IGCM_TMP}/${$}.liste_output_files_config.txt ) >> ${IGCM_TMP}/${$}.liste_no_output_files_config.txt
        fi
    done
    if [ -f ${IGCM_TMP}/${$}.liste_no_output_files_config.txt ]; then 
        join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.liste_no_output_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt
        mv ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.liste_output_files_config.txt
    fi

    # Lecture du nom du job
    JobName=`awk -F/ '{if (NR==1) print $(NF-2)}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt`

    DEM_log -2 JobName ${JobName}

   # traitons les listes de restart pour conserver ceux qui correspondent au format $JobName_restart_date_date.tar et $JobName_date_date_restart.tar
   # on enleve les fichiers qui ne sont pas en .tar
   grep -w "tar" ${IGCM_TMP}/${$}.liste_restart_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt; mv ${IGCM_TMP}/${$}.info_tmp.txt  ${IGCM_TMP}/${$}.liste_restart_files_config.txt 
   # dans ces fichiers on enleve ceux qui n'ont pas la chaine "restart" (attention suivant d'ou provient la simulation (pack ou libigcm) la chaine restart n'est pas situee au meme endroite
   grep "restart"  ${IGCM_TMP}/${$}.liste_restart_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt; mv ${IGCM_TMP}/${$}.info_tmp.txt  ${IGCM_TMP}/${$}.liste_restart_files_config.txt 
   # dans ces fichiers on enleve ceux qui n'ont pas le jobName
   grep $JobName ${IGCM_TMP}/${$}.liste_restart_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt; mv ${IGCM_TMP}/${$}.info_tmp.txt  ${IGCM_TMP}/${$}.liste_restart_files_config.txt    

   mv ${IGCM_TMP}/${$}.liste_restart_files_config.txt    ${IGCM_DEM_SIMU}/liste_restart_files_config.txt

# Maintenant que nous avons JobName verifions que tous les fichiers de Output sont bien du type JobName_date_*
    sed -e "s%.*/${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt
    cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" >  ${IGCM_TMP}/${$}.info_tmp_bis.txt
    join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt > ${IGCM_TMP}/${$}.output.tmp
    sort -k 2 ${IGCM_TMP}/${$}.output.tmp > ${IGCM_TMP}/${$}.output_bis.tmp
    mv ${IGCM_TMP}/${$}.output_bis.tmp ${IGCM_DEM_SIMU}/liste_output_files_config.txt


## Recherche de DateBegin et DateEnd pour les Restarts
    typeRestart=`awk -F_ '{if (NR==1) print $(NF)}'  ${IGCM_DEM_SIMU}/liste_restart_files_config.txt `
    echo $typeRestart
    if [[ "$typeRestart" == "restart.tar" ]]; then 
        first_file=$(        head -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
        extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_//" | sed -e 's/_restart.tar//g' )
        DateBegin_Restart=$( echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )

        last_file=$(         tail -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
        extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" | sed -e 's/_restart.tar//g' )
        DateEnd_Restart=$(   echo ${extract_date_file}| sed 's/.*_\([0-9]\{8\}\).*$/\1/g' )
    else
        first_file=$(        head -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
        extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_restart_//" | sed -e 's/\.tar//g' )
        DateBegin_Restart=$( echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )

        last_file=$(         tail -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
        extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_restart_//" | sed -e 's/\.tar//g' )
        DateEnd_Restart=$(   echo ${extract_date_file}| sed 's/.*_\([0-9]\{8\}\).*$/\1/g' )
        
    fi


# Recherche de DateBegin et DateEnd pour les Output
    first_file=$( head -n 1 ${IGCM_DEM_SIMU}/liste_output_files_config.txt )
    extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_//" )
    DateBegin_Output=$(  echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
    
    last_file=$(         tail -n 1 ${IGCM_DEM_SIMU}/liste_output_files_config.txt )
    extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" )
    DateEnd_Output=$(    echo ${extract_date_file}| sed 's/.*_\([0-9]\{8\}\)_.*$/\1/g' )
    
    DateBegin=${DateBegin_Restart}
    DateEnd=${DateEnd_Restart}

    DEM_log -2 "DateBegin DateEnd : ${DateBegin} ${DateEnd}"
    DEM_log -2 "DateBegin_Output DateEnd_Output : ${DateBegin_Output} ${DateEnd_Output}"
    DEM_log -2 "DateBegin_Restart DateEnd_Restart : ${DateBegin_Restart} ${DateEnd_Restart}"

    DateBegin=$( DEM_min ${DateBegin_Output} ${DateBegin} ) 
    DateEnd=$(   DEM_max ${DateEnd_Output}   ${DateEnd}   )
    
    DEM_log -2  "DateBegin DateEnd : ${DateBegin} ${DateEnd}"
    
    [[ "${clean_tmp}" = "yes" ]] && rm -f ${IGCM_TMP}/${$}.config_name ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.liste_restart_files_config.txt
    
    rm -f ${IGCM_DEM_SIMU}/config_card_${JobName}
    cat<<EOD>${IGCM_DEM_SIMU}/config_card_${JobName}
JobName=${JobName}
DateBegin=${DateBegin}
DateEnd=${DateEnd}
PATH_SIMUL_FULL=${PATH_SIMUL_FULL}
IGCM_DEM_SIMU=${IGCM_DEM_SIMU}
EOD
    
    echo "${IGCM_DEM_SIMU}/config_card_${JobName} ListToBeDone" >> ${IGCM_DEM}/config_card.liste

    DEM_log -1 "${IGCM_DEM_SIMU}/config_card_${JobName} ListToBeDone >> ${IGCM_DEM}/config_card.liste"

#si la liste des fichiers d'output est vide on tar la simul 
if [[ !  -s ${IGCM_DEM_SIMU}/liste_output_files_config.txt ]]; then
DEM_log -2 "liste output vide"
    cd ${EXE_DIR}
    ./write_liste_tar.sh ${IGCM_DEM_SIMU}/config_card_${JobName}
    cd ${JOB_DIR}
fi 


done

# gpdebug : le script n'a pas rencontre d'erreur. On l'inscrit dans le fichier de suivi
echo $MonitoringInfo >> $generalMonitorFile

DEM_log -0 "Fin de ${SCRIPT_NAME}"