#!/bin/ksh

#**************************************************************
# Author: Jacques Belier
# Contact:
# $Revision::                                          $ Revision of last commit
# $Author::                                            $ Author of last commit
# $Date::                                              $ Date of last commit
# IPSL (2006)
#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
#
#**************************************************************
#---------------------------------------------------------------------
#- Installation of jobs according to environment
#---------------------------------------------------------------------
function ins_job_Usage
{
print - "
ins_job installs the jobs in the directories
which contain a file config.card

ins_job must be launched on the host
on which the job will be submitted

Usage :
  ${b_n} [-h] [-v] [-e] 
  or on irene/TGCC :
  ${b_n} [-h] [-v] [-e] [-p project] [-q type_of_node] [-c number of cores]
  or on ada /IDRIS
  ${b_n} [-h] [-v] [-e] [-m MPI environment]
Options :
  -h                  : help
  -v                  : verbose mode
  -e                  : turn on ensemble mode (hindcast/forecast or date restart)
  -f                  : ins_job is force to create jobs even if they already exist
on irene only : 
  -p project          : add default project on irene
  -q type_of_node     : add default type of nodes for postprocessing on irene skylake/xlarge 
  -c number of cores  : add default number of cores for postprocessing on irene 1-112
on ada only :
  - m MPI environment : add default MPI environment (Intel MPI or IBM MPI)
  -t wall time        : change default wall time
"
}
function ins_job_Warning
{
   [[ ${x_v} = 'verbose' ]] && print - "\n############### WARNING ###############";
   [[ ${x_v} = 'verbose' ]] && print - "File ${n_f} already exists\nin directory ${j}";
   [[ ${x_v} = 'verbose' ]] && print - "You must delete this file to update !";
}

function ins_job_Check_JobName
{
  verif=${JobName##[a-zA-Z]*(?([.\-])[a-zA-Z0-9])}

  if [ ${#verif} -ne 0 ] ; then
    echo "################ ERROR ################"
    echo "${JobName} is invalid."
    echo "- JobName can only contain alphanumeric characters, \".\" and \"-\""
    echo "- JobName must start with a letter"

    ((NbErr=NbErr+1))

    Status=1
  else
    Status=0
  fi

  return ${Status}
}

#-
#     dirname     and      basename
#-
d_n=$(dirname ${0}); b_n=$(basename ${0});
#-
# Retrieving and validation of the options
#-
x_v='silencious';
x_e=false;
x_f=false;
x_p=false;
x_q=false;
x_c=false;
x_m=false;
x_t=false;
while getopts :hvefc:p:m:q:s:t: V ; do
  case $V in
  (h)  ins_job_Usage; exit 0;;
  (v)  x_v='verbose';;
  (e)  x_e=true;;
  (f)  x_f=true;;
  (p)  x_p=true
       ProjectID=${OPTARG} ;;
  (q)  x_q=true
       ProjectNode=${OPTARG} ;;
  (c)  x_c=true
       ProjectCore=${OPTARG} ;;
  (m)  x_m=true
       MPIEnvironment=${OPTARG} ;;
  (t)  x_t=true
       WallTime=${OPTARG} ;;
  (:)  echo ${b_n}" : -"${OPTARG}" option : missing value" 1>&2;
       exit 2;;
  (\?) echo ${b_n}" : -"${OPTARG}" option : not supported" 1>&2;
       exit 2;;
  esac
done
[ ${x_v} = 'silencious' ] && export DEBUG_sys=false
shift $(($OPTIND-1));
#-
# Define working files
#-
F_MOD=$(cd ${d_n}'/..';/bin/pwd;)
# [[ ${F_MOD##*/} != 'modipsl' ]] && \
#  { print - "directory 'modipsl' unreachable"; exit 3; }
W_W=${d_n}'/../libIGCM'
#W_W=${d_n}  # S.Senesi - not sure of the intent in line above, but that clashes with using a symbolic link for libIGCM in modipsl/, which is must useful for development
[[ ! -d ${W_W} ]] && { print - "${W_W} unreachable"; exit 3; }
libIGCM=$(cd ${W_W};/bin/pwd;)
F_JOB=${libIGCM}'/AA_job';
[[ ! -f ${F_JOB} ]] && { print - "${F_JOB} unreachable"; exit 3; }
F_JOB_DEBUG=${libIGCM}'/AA_job_debug';
[[ ! -f ${F_JOB_DEBUG} ]] && { print - "${F_JOB_DEBUG} unreachable"; exit 3; }
F_RCI=${libIGCM}'/run.card.init';
[[ ! -f ${F_RCI} ]] && { print - "${F_RCI} unreachable"; exit 3; }
#-
# Accessing to functions (without stack)
#-
# No verbosity (0, 1, 2, 3)
Verbosity=0
# No de debug
DEBUG_debug=false
# Dont move libIGCM
MirrorlibIGCM=false
# Behave like computing job
TaskType=computing
# Source libIGCM
. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
if [ $x_e = 'true' ] ; then
  . ${libIGCM}/libIGCM_ensemble/libIGCM_ensemble.ksh
fi

#-
[[ ${x_v} = 'verbose' ]] && \
 {
  print - "";
  print - '--- Host        : '${SYSTEM};
  print - '--- modipsl     : '${F_MOD};
  print - '--- libIGCM     : '${libIGCM};
  print - '--- basic job   : '${F_JOB};
  print - '--- basic card  : '${F_RCI};
 }
#-
[[ ${x_v} = 'verbose' ]] && print - "\nInstallation of jobs for '${SYSTEM}'";
#-

NbErr=0

#-
# Define Project parameters to set up jobs header for Irene (TGCC)
# on Irene define ProjectID and ProjectCore : option or answer
# on Irene ProjectNode default projectnode set to xlarge since 17/10/2018
#-


if [ X"${SYSTEM}" == "Xirene" ] ; then

  #- set ProjectID if required
  if ( ! ${x_p} ) ; then
    print - "Wait for the next question ..."
    #- default ProjectID
    ProjectID=$( ccc_myproject | grep -i irene |grep -i skylake | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | sort -u | grep -v gencmip6 | head -n 1 )
    answer=""
    print - "Hit Enter or give project ID (default is ${ProjectID}), possible projects are $( echo $( ccc_myproject | grep -i irene | grep -i skylake | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | grep -v gencmip6 | sort -u ) ) or other xxxcmip6 : $(for i in $(groups) ; do echo $i|grep -v gencmip6|grep .cmip6 1>/dev/null 2>&1 && echo -n $i " " ; done ; echo ) "
    read answer

    if [ "X${answer}" != "X" ] ; then
      ProjectID=${answer}
    fi

  fi # if ( ! ${x_p} )

  echo  ProjectID is ${ProjectID} at Irene 


  # set ProjectNode if required
  if ( ! ${x_q} ) ; then 
     #- default ProjectNode 
     ProjectNode="xlarge"
     #- is xlarge possible for ${ProjectID} ?
     echo $(ccc_myproject | grep -i irene | grep -i xlarge | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | grep -v gencmip6 | sort -u) $(for i in $(groups) ; do echo $i|grep -v gencmip6|grep .cmip6 1>/dev/null 2>&1 && echo -n $i " " ; done ) | grep ${ProjectID} >/dev/null 2>&1  || ProjectNode="xlarge"

     if [ "X${ProjectNode}" == "Xxlarge" ] ; then
       answerOK=false 
  
       while ( ! ${answerOK} ) ; do 
         answer="" 
         print - "Hit Enter or give TYPE OF NODE required for post-processing (default is \"${ProjectNode}\"), possible types of nodes are \"skylake\" or \"xlarge\" : " 
         read answer 
         [ "X${answer}" == "X" ] || [ "X${answer}" == "Xskylake" ] || [ "X${answer}" == "Xxlarge" ] && answerOK=true 
       done 
  
       if [ "X${answer}" != "X" ] ; then 
         ProjectNode=${answer} 
       fi 

     else
       print - "PostProcessing will be done on ${ProjectNode}"
     fi
  
  fi # if ( ! ${x_q} ) 
  echo ProjectNode for post-processing is ${ProjectNode} at Irene 

  #- ProjectNode is known (option or answer) set ProjectCoreMax 
  [ "${ProjectNode}" = "xlarge" ] && ProjectCoreMax="112" || ProjectCoreMax="48" 
  #- ProjectCoreMax is 48 for standard and 112 for xlarge

  if ( ! ${x_c} ) ; then
    #- ProjectNode is known (option or answer), set ProjectCore default
    [ "${ProjectNode}" = "xlarge" ] && ProjectCore="8" || ProjectCore="4" 

    # let check minimum/maximum value 1/${ProjectCoreMax}

    answerOK=false

    while ( ! ${answerOK} ) ; do
      answer=""
      print - "possible numbers of cores are \"1\" to \"${ProjectCoreMax}\" for ${ProjectNode} : "
      print - "Hit Enter or give NUMBER OF CORES required for post-processing (default is \"${ProjectCore}\")"
      read answer
      [ "X${answer}" == "X" ] || [ ${answer} -ge 1 -a ${answer} -le ${ProjectCoreMax} ] && answerOK=true
    done

    if [ "X${answer}" != "X" ] ; then
      ProjectCore=${answer}
    fi

  fi # if ( ! ${x_c} )

  echo ProjectCore for post-processing is ${ProjectCore}
  #- ProjectCore is set (option or answer)

  #- set WallTime if required
  if ( ! ${x_t} ) ; then
    #- default WallTime 1800 s 
    WallTime=1800
    answer=""
    print - "Hit Enter or give required wall time in seconds for computing job (default is \"${WallTime}\" seconds, maximum is 86400 seconds) "
    read answer

    if [ "X${answer}" != "X" ] ; then
      WallTime=${answer}
    fi

  fi # if ( ! ${x_t} )
   
  echo Wall time limit is ${WallTime} seconds

elif [ X"${SYSTEM}" == "Xirene-amd" ] ; then

  #- set ProjectID if required
  if ( ! ${x_p} ) ; then
    print - "Wait for the next question ..."
    #- default ProjectID
    ProjectID=$( ccc_myproject | grep -i irene |grep -i rome | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | sort -u | grep -v gencmip6 | head -n 1 )
    answer=""
    print - "Hit Enter or give project ID (default is ${ProjectID}), possible projects are $( echo $( ccc_myproject | grep -i irene | grep -i rome | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | grep -v gencmip6 | sort -u ) ) or other xxxcmip6 : $(for i in $(groups) ; do echo $i|grep -v gencmip6|grep .cmip6 1>/dev/null 2>&1 && echo -n $i " " ; done ; echo ) "
    read answer

    if [ "X${answer}" != "X" ] ; then
      ProjectID=${answer}
    fi

  fi # if ( ! ${x_p} )

  echo  ProjectID is ${ProjectID} at Irene-amd

  # set ProjectNode if required
  if ( ! ${x_q} ) ; then 
     #- default ProjectNode 
     ProjectNode="rome"
     #- is xlarge possible for ${ProjectID} ?
     echo $(ccc_myproject | grep -i irene | grep -i xlarge | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | grep -v gencmip6 | sort -u) $(for i in $(groups) ; do echo $i|grep -v gencmip6|grep .cmip6 1>/dev/null 2>&1 && echo -n $i " " ; done ) | grep ${ProjectID} >/dev/null 2>&1  || ProjectNode="rome"

     if [ "X${ProjectNode}" == "Xxlarge" ] ; then
       answerOK=false 
  
       while ( ! ${answerOK} ) ; do 
         answer="" 
         print - "Hit Enter or give TYPE OF NODE required for post-processing (default is \"${ProjectNode}\"), possible types of nodes are \"rome\"  : " 
         read answer 
         [ "X${answer}" == "X" ] || [ "X${answer}" == "Xrome" ] && answerOK=true 
       done 
  
       if [ "X${answer}" != "X" ] ; then 
         ProjectNode=${answer} 
       fi 

     else
       print - "PostProcessing will be done on ${ProjectNode}"
     fi
  
  fi # if ( ! ${x_q} ) 
  echo ProjectNode for post-processing is ${ProjectNode} at Irene-amd

  #- ProjectNode is known (option or answer) set ProjectCoreMax 
  [ "${ProjectNode}" = "xlarge" ] && ProjectCoreMax="112" || ProjectCoreMax="48" 
  #- ProjectCoreMax is 48 for standard and 112 for xlarge

  if ( ! ${x_c} ) ; then
    #- ProjectNode is known (option or answer), set ProjectCore default
    [ "${ProjectNode}" = "xlarge" ] && ProjectCore="8" || ProjectCore="4" 

    # let check minimum/maximum value 1/${ProjectCoreMax}

    answerOK=false

    while ( ! ${answerOK} ) ; do
      answer=""
      print - "possible numbers of cores are \"1\" to \"${ProjectCoreMax}\" for ${ProjectNode} : "
      print - "Hit Enter or give NUMBER OF CORES required for post-processing (default is \"${ProjectCore}\")"
      read answer
      [ "X${answer}" == "X" ] || [ ${answer} -ge 1 -a ${answer} -le ${ProjectCoreMax} ] && answerOK=true
    done

    if [ "X${answer}" != "X" ] ; then
      ProjectCore=${answer}
    fi

  fi # if ( ! ${x_c} )

  echo ProjectCore for post-processing is ${ProjectCore}
  #- ProjectCore is set (option or answer)

  #- set WallTime if required
  if ( ! ${x_t} ) ; then
    #- default WallTime 1800 s 
    WallTime=1800
    answer=""
    print - "Hit Enter or give required wall time in seconds for computing job (default is \"${WallTime}\" seconds, maximum is 86400 seconds) "
    read answer

    if [ "X${answer}" != "X" ] ; then
      WallTime=${answer}
    fi

  fi # if ( ! ${x_t} )
   
  echo Wall time limit is ${WallTime} seconds

elif [ X"${SYSTEM}" == "Xjeanzay" ] ; then

  #- set ProjectID if required
  if ( ! ${x_p} ) ; then
    print - "Wait for the next question ..."
    #- default ProjectID
    ProjectID=$( echo $IDRPROJ )
    answer=""
    print - "Hit Enter or give project ID (default is ${ProjectID}), possible projects are: $( groups ; echo ) "
    read answer

    if [ "X${answer}" != "X" ] ; then
      ProjectID=${answer}
    fi

  fi # if ( ! ${x_p} )

  echo  ProjectID is ${ProjectID} at Jean-Zay

  #- set WallTime if required
  if ( ! ${x_t} ) ; then
    #- default WallTime 30 minutes
    WallTime=30
    answer=""
    print - "Hit Enter or give required wall time in minutes for computing job (default is \"${WallTime}\" minutes, maximum is 1200 minutes) "
    read answer

    if [ "X${answer}" != "X" ] ; then
      WallTime=${answer}
    fi

  fi # if ( ! ${x_t} )
   
  echo Wall time limit is ${WallTime} minutes

elif [ X"${SYSTEM}" == "Xada" ] ; then
  if ( ! ${x_m} ) ; then
    MPIEnvironment=IBM 
    # Intel MPI Environment.
    answerOK=false   	
    while ( ! ${answerOK} ) ; do
      answer=""
      print - "Hit Enter or give MPI Environement (default is ${MPIEnvironment}), possible MPI environments are IBM (MPI IBM) and Intel (MPI Intel) :"
      read answer
      [ "X${answer}" == "X" ] || [ "X${answer}" == "XIBM" ] || [ "X${answer}" == "XIntel" ] && answerOK=true
    done

    if [ "X${answer}" != "X" ] ; then
      MPIEnvironment=${answer}
    fi
  fi # if ( ! ${x_pm} )
  echo MPIEnvironment is ${MPIEnvironment}

elif [ X"${SYSTEM}" == "Xobelix" ] || [ X"${SYSTEM}" == "Xifort_CICLAD" ] ; then
  # obelix, ciclad, climserv
  echo ""
  echo "You need to check and maybe adapt headers in the main job especially the line: "
  echo "  #PBS -l nodes=x:ppn=y "
  echo "where x is the number of nodes, y the number of cores per node and x*y is the total number of cores for the job. "
  echo "y must not be bigger than the maximum numer of cores per node on the machine (often 8 or 16)."
fi # if [ X"${SYSTEM}" == "Xirene" ]

#-
# Define the pattern string to substitute
#-
W_P='#-Q- '; W_W=${W_P}${SYSTEM}' ';
#-
# Extract list of 'config.card' files
# and create jobs with AA_job
#-
F_CFG='config.card';
F_CFG_ENS='ensemble.card';
SUBMIT_DIR_ENS=$( pwd )
for i in $( pwd )/config.card 
do
  if [ ! -f $i ] ; then
    echo ""
    echo "################## WARNING ##################"
    echo "No config.card available in current directory"
    echo ""
    continue 
  fi 


  j=$(cd ${i%/*};/bin/pwd;)
  n_f=${F_RCI##*/};

  if [ ! X$( echo ${j} | grep EXPERIMENTS ) = X ] ; then
    # Do not treat config.card if it is in sub-directory of EXPERIMENTS
    # Continue to next config.card
    continue
  else
    [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
  fi

  # Find out if new structure and set .resol filename
  if [ -d ${j}/EXPERIMENTS ] && [ -d ${j}/GENERAL ] ; then
    # New Structure
    [[ ${x_v} = 'verbose' ]] && echo "This is new configuration structure"
    new_struct=yes
    resolfile=$j/.resol
  else
    # Old Structure
    new_struct=no
    resolfile=$j/../.resol
  fi

  # Get all variables declared in section UserChoices in config.card
  IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} UserChoices
  # Set default values
  config_UserChoices_ExpType=""
  RESOL_ATM_3D=this_is_a_test_string
  RESOL=this_is_another_test_string
  ResolAtm=this_is_a_new_another_test
  typeset option
  for option in ${config_UserChoices[*]} ; do
    IGCM_card_DefineVariableFromOption ${j}'/'${F_CFG} UserChoices ${option}
  done

  # Find the JobName : JobName might contain the variable RESOL_ATM_3D that will be replaced by what is in .resol file
  if [ ! X$( echo ${config_UserChoices_JobName} | grep ${RESOL_ATM_3D} ) = X ] ; then
    TRUERESOL=$( tail -1 $resolfile | awk "-F=" '{print $2}' )
    echo TRUERESOL = $TRUERESOL
    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${RESOL_ATM_3D}/${TRUERESOL}/" )
    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
  elif [ ! X$( echo ${config_UserChoices_JobName} | grep ${RESOL} ) = X ] ; then
    TRUERESOL=$( head -1 $resolfile  )
    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${RESOL}/${TRUERESOL}/" )
    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
  elif [ ! X$( echo ${config_UserChoices_JobName} | grep ${ResolAtm} ) = X ] ; then
    TRUERESOL=${config_UserChoices_ResolAtm}
    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${ResolAtm}/${TRUERESOL}/" )
    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
  else
    JobName=${config_UserChoices_JobName}
  fi

  # Check JobName validity : only alphanumerical characters, "-" and "." are authorized
  ins_job_Check_JobName
  RetCode=$?
  [[ $RetCode -gt 0 ]] && continue

  [[ ${x_v} = 'verbose' ]] && echo "JobName=${JobName}"

  # Add specific treatment for new type of directory structure
  if [ ${new_struct} == yes ] ; then

    if [ "X${config_UserChoices_ExpType}" = X ] ; then
      echo "\nERROR in ${j}/config.card"
      echo "ins_job stops here"
      echo "=> The variable ExpType must be added in config.card in section UserChoices"
      echo "=> ExpType gives the directory for the .card configuration files for the wanted experiement. For exemple ExpType=IPSLCM5/historical"
      exit 4
    else
      [[ ${x_v} = 'verbose' ]] && echo "ExpType= ${config_UserChoices_ExpType}"
    fi

    if [ -d ${j}/${JobName} ] ; then
      echo "Directory ${j}/${JobName} exists already. It will not be overwritten."
      echo "Remove the existing directory or change JobName before relaunching ins_job."
      #continue
      exit  
    fi
    echo "=> Submit directory ${JobName} will be created with cards from EXPERIMENTS/${config_UserChoices_ExpType}"
    cp -r ${j}/EXPERIMENTS/${config_UserChoices_ExpType} ${j}/${JobName}
    cp -r ${j}/GENERAL/* ${j}/${JobName}/.
    cp -f ${j}/${F_CFG}  ${j}/${JobName}/.
    if [ -f ${F_CFG_ENS} ] ; then
      cp -f ${j}/${F_CFG_ENS}  ${j}/${JobName}/.
      SUBMIT_DIR_ENS=${j}/${JobName}
    fi
    rm -f ${j}/${F_CFG}
    # rm -f ${j}/${F_CFG_ENS}
    rm -f ${j}/${F_CFG}.bak
    j=${j}/${JobName}
    [[ ${x_v} = 'verbose' ]] && echo new j=$j
  fi
  # end specific treatment for new type directory structure

  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; } || \
   {
    [[ ${x_v} = 'verbose' ]] && print - "\nCopying file ${F_RCI}\nin directory ${j}";
    \cp ${F_RCI} ${j};
   }

  #==================================
  # Read ListOfComponents section:
  #echo
  #IGCM_debug_Print 1 "DefineArrayFromSection : ListOfComponents"

  IGCM_card_DefineArrayFromSection  ${j}'/'${F_CFG} ListOfComponents
  for comp in ${config_ListOfComponents[*]} ; do
    IGCM_card_DefineArrayFromOption  ${j}'/'${F_CFG} ListOfComponents ${comp}
  done
  #IGCM_debug_Print 3 ${config_ListOfComponents[*]}

  #==================================
  # Read Executable section:
  IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} Executable

  # Define the execution context (MPMD, SPMD, MPI/OMP ...)
  IGCM_config_ConfigureExecution ${j}'/'${F_CFG}

  # coreNumber    : TOTAL NUMBER OF CORES
  # mpiTasks      : TOTAL NUMBER OF MPI TASKS
  # openMPthreads : NUMBER OF OpenMP THREADS

  # File name for Job_debug
  n_f='Job_debug_'${JobName};
  [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
  sed -e "/^${W_W} */ s///" \
      -e "/^${W_P}/d"       \
      -e "s%::modipsl::%${F_MOD}%" \
      -e "s/::Jobname::/${JobName}/" \
      -e "s/::default_project::/${ProjectID}/" \
      -e "s/::WallTime::/${WallTime}/" \
      ${F_JOB_DEBUG} > ${libIGCM}'/'${n_f}
  chmod u+x ${libIGCM}'/'${n_f}

  # update Headers so that ressources description are accurate (MPMD/SPMD/...)
  IGCM_sys_updateHeaders ${libIGCM}'/'${n_f}

  # File name for Job
  n_f='Job_'${JobName};
  [[ ${x_f} = 'false' ]] && [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
  [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
  sed -e "/^${W_W} */ s///" \
      -e "/^${W_P}/d"       \
      -e "s%::modipsl::%${F_MOD}%" \
      -e "s/::Jobname::/${JobName}/" \
      -e "s/::default_project::/${ProjectID}/" \
      -e "s/::WallTime::/${WallTime}/" \
      ${F_JOB} > ${j}'/'${n_f}
  chmod u+x ${j}'/'${n_f}

  # update Headers so that ressources description are accurate (MPMD/SPMD/...)
  IGCM_sys_updateHeaders ${j}'/'${n_f}
  
done

#-
# Extract list of AA_* files in libIGCM
# and create jobs (for all except AA_job)
#-
for i in $(find ${libIGCM} -maxdepth 1 -name "AA_*" -print)
do
  i_f=${i##*/};
  [[ ${i_f} = 'AA_job' ]] && { continue; }
  [[ ${i_f} = 'AA_job_debug' ]]  && { continue; }
  j=${i%/*}; n_f=${i_f#AA_}'.job';
  [[ ${x_f} = 'false' ]] && [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
  [[ ${x_v} = 'verbose' ]] && print - "\nIn directory ${j}\n${i_f} -> ${n_f}"
  sed -e "/^${W_W} */ s///" \
      -e "s%::modipsl::%${F_MOD}%" \
      -e "/^${W_P}/d"       \
      -e "s/::default_node::/${ProjectNode}/" \
      -e "s/::default_core::/${ProjectCore}/" \
      -e "s/::default_project::/${ProjectID}/" \
      -e "s/::default_post_project::/${ProjectID}/" \
      ${i} > ${j}'/'${n_f}
  chmod u+x ${j}'/'${n_f}
done
#-
# set default_project in libIGCM_sys_irene and libIGCM_sys_jeanzay.
#-
if [ X"${SYSTEM}" == "Xirene" ] ; then
  i=${libIGCM}/libIGCM_sys/libIGCM_sys_irene.ksh
  sed -i -e "s/::default_project::/${ProjectID}/" ${i}
elif [ X"${SYSTEM}" == "Xirene-amd" ] ; then
  i=${libIGCM}/libIGCM_sys/libIGCM_sys_irene-amd.ksh
  sed -i -e "s/::default_project::/${ProjectID}/" ${i}
elif [ X"${SYSTEM}" == "Xjeanzay" ] ; then
  i=${libIGCM}/libIGCM_sys/libIGCM_sys_jeanzay.ksh
  sed -i -e "s/::default_project::/${ProjectID}/" ${i}
fi
#-
# Limited to hindcast/forecast and date restart Ensemble for the time being
if [ ${x_e} = 'true' ] ; then
    if [ ! -f ${F_CFG_ENS} ] ; then
       echo ""
       echo "################## WARNING ##################"
       echo "No ensemble.card available in current directory"
       echo ""
       exit
      #  continue
   fi

  #.. Read input data from ensemble.card ..
  SUBMIT_DIR=${SUBMIT_DIR_ENS}
  RUN_DIR="${CCCWORKDIR}/ENSEMBLE_TMP"
  #
  # Copy initial things around and define variables (hindcast/forecast case)
  IGCM_sys_Cd ${SUBMIT_DIR}
  IGCM_ensemble_Init

  if [[ ${ensemble_Ens_DATE_active} = 'y' ]] ; then
    IGCM_sys_Cd ${SUBMIT_DIR}
    IGCM_ensemble_DateInit
    # As it says
    IGCM_sys_Cd ${SUBMIT_DIR}
    IGCM_ensemble_DateNonPeriodicStarts
  fi

  if [[ ${ensemble_Ens_PERTURB_active} = 'y' ]] ; then
    IGCM_sys_Cd ${SUBMIT_DIR}
    IGCM_ensemble_CastInit
    # As it says
    IGCM_sys_Cd ${SUBMIT_DIR}
    IGCM_ensemble_CastPeriodicStarts
    # As it says
    #IGCM_sys_Cd ${SUBMIT_DIR}
    #IGCM_ensemble_CastMemberList
  fi
  # Done
  IGCM_sys_Cd ${SUBMIT_DIR}
  # Clean
  IGCM_sys_Rm -rf ${RUN_DIR}
  
fi
#
# Install a light copy of C-ESM-EP in SUBMIT_DIR/Cesmep and create a
# C-ESM-EP launch script there.
#
[ ${new_struct?} == yes ] && prefix="$JobName/"  
IGCM_card_DefineVariableFromOption ${prefix}config.card Post Cesmep
if [ ${config_Post_Cesmep} != FALSE ]; then
    IGCM_config_CommonConfiguration ${prefix}config.card
    if [ x"${config_Post_CesmepMail}" == x"TRUE" ] || [ x"${config_Post_CesmepMail}" == x"True" ] ; then
	# Compute mail adress - inspired by libIGCM_post.ksh - should be turned in a function 
	if [ ! -z ${config_UserChoices_MailName} ] ; then
	    MailAdress=${config_UserChoices_MailName}
	elif [ -f ~/.forward ] ; then
	    MailAdress=$( cat ~/.forward )
	else
	    MailAdress=${USER}
	fi
    else
	MailAdress=None
    fi
    #
    components=","
    for comp in ${config_ListOfComponents[*]} ; do
	components=${components}${comp}, ;
    done
    if [ X"${SYSTEM}" == "Xirene" ] || [ X"${SYSTEM}" == "Xirene-amd" ] ; then
	IGCM_card_DefineVariableFromOption ${prefix}config.card UserChoices DateBegin
	${config_Post_CesmepCode}/libIGCM_install.sh $(pwd)/${prefix} \
		   ${config_Post_CesmepComparison} $JobName ${R_SAVE} ${ProjectID} \
		   ${MailAdress} ${config_UserChoices_DateBegin//-/} \
		   ${config_Post_Cesmep} ${CesmepPeriod} ${config_Post_CesmepSlices} $components
   else
       echo -e "\nERROR Cannot (yet) handle CESMEP atlas on this computer system"
       echo "=> Variable Cesmep in config.card's section Post must be set to FALSE"
       exit 5
   fi
fi
#-
[[ ${x_v} = 'verbose' ]] && print - "";
#-
# That's all folks
#-

if [ ${NbErr} -ne 0 ] ; then
  echo "################ ERROR ################"
  echo "${NbErr} invalid JobName(s) found, check the log"
fi


exit 0;