Changeset 1490 for trunk/libIGCM


Ignore:
Timestamp:
10/16/19 10:44:35 (5 years ago)
Author:
jgipsl
Message:

Added first version of sys_jeanzay. Still lots to do. Only case done for now is one executable mpi mode. Only create_ts done among post-treatments and only from main job (not done via TimeSeriesChecker?). Change of data project is not working (follwoing is not working when launched from job: eval $(idrenv -d ${PROJECT}) ).

Location:
trunk/libIGCM
Files:
4 edited
1 copied

Legend:

Unmodified
Added
Removed

  • trunk/libIGCM/AA_create_ts

    r1468 r1490  
    2626#-Q- irene #MSUB -m store,work,scratch 
    2727#-Q- irene set +x 
     28#-Q- jeanzay #!/bin/ksh  
     29#-Q- jeanzay ###################### 
     30#-Q- jeanzay ## JEANZAY    IDRIS ## 
     31#-Q- jeanzay ###################### 
     32#-Q- jeanzay #SBATCH --job-name=TS         # Job Name 
     33#-Q- jeanzay #SBATCH --output=TS.out_%I    # standard output 
     34#-Q- jeanzay #SBATCH --error=TS.out_%I     # error output 
     35#-Q- jeanzay #SBATCH --ntasks=1            # Number of core 
     36#-Q- jeanzay #SBATCH --hint=nomultithread  # 1 processus MPI par coeur physique (pas d'hyperthreading) 
     37#-Q- jeanzay #SBATCH --time=10:00:00       # Wall clock limit (seconds) 
     38#-Q- jeanzay #SBATCH --account ::default_project::@cpu 
     39#-Q- jeanzay set +x 
    2840#-Q- ada #!/bin/ksh 
    2941#-Q- ada ####################### 

  • trunk/libIGCM/AA_job

    r1460 r1490  
    4343#-Q- irene BATCH_NUM_PROC_TOT=$BRIDGE_MSUB_NPROC 
    4444#-Q- irene set +x 
     45#-Q- jeanzay #!/bin/ksh 
     46#-Q- jeanzay ###################### 
     47#-Q- jeanzay ## JEANZAY    IDRIS ## 
     48#-Q- jeanzay ###################### 
     49#-Q- jeanzay #SBATCH --job-name=::Jobname::        # Job Name 
     50#-Q- jeanzay #SBATCH --output=Script_Output_::Jobname::.000001    # standard output 
     51#-Q- jeanzay #SBATCH --error=Script_Output_::Jobname::.000001    # error output 
     52#-Q- jeanzay ####SBATCH -eo 
     53#-Q- jeanzay #SBATCH --ntasks=::JobNumProcTot::   # Number of MPI tasks 
     54#-Q- jeanzay #SBATCH --cpus-per-task=::openMPthreads::  # Number of openMP threads. 
     55#-Q- jeanzay #SBATCH --hint=nomultithread         # 1 processus MPI par coeur physique (pas d'hyperthreading) 
     56#-Q- jeanzay #SBATCH --time=00:30:00              # Wall clock limit (seconds) 
     57#-Q- jeanzay #SBATCH --account ::default_project::@cpu 
     58#-Q- jeanzay 
     59#-Q- jeanzay ##BATCH_NUM_PROC_TOT=$BRIDGE_SBATCH_NPROC 
     60#-Q- jeanzay set +x 
    4561#-Q- ada #!/bin/ksh 
    4662#-Q- ada # ###################### 
    … …  
    189205#SUBMIT_DIR=$( pwd ) 
    190206 
    191 #D- Define input file root directory 
    192 #D- Default=/IGCMG/common/account/of/this/machine 
     207#D- Define input file root directory R_IN 
    193208#D- IPSL:   /prodigfs/ipslfs/igcmg/IGCM 
    194209#D- TGCC:   /ccc/work/cont003/igcmg/igcmg/IGCM 
    195 #D- IDRIS:  /workgpfs/rech/psl/rpsl035/IGCM 
     210#D- IDRIS:  /gpfswork/rech/psl/commun/IGCM 
    196211#D- OBELIX: /home/orchideeshare/igcmg/IGCM 
    197 #D- OTHER:  /home/${LOGIN}/IGCM 
     212#D- Other default:  /home_local/${LOGIN}/IGCM 
    198213 
    199214#D- Change below to override the default 

  • trunk/libIGCM/ins_job

    r1489 r1490  
    239239  #- ProjectCore is set (option or answer) 
    240240 
     241elif [ X"${SYSTEM}" == "Xjeanzay" ] ; then 
     242 
     243  #- set ProjectID if required 
     244  if ( ! ${x_p} ) ; then 
     245    print - "Wait for the next question ..." 
     246    #- default ProjectID 
     247    ProjectID=$( echo $IDRPROJ ) 
     248    answer="" 
     249    print - "Hit Enter or give project ID (default is ${ProjectID}), possible projects are: $( groups ; echo ) " 
     250    read answer 
     251 
     252    if [ "X${answer}" != "X" ] ; then 
     253      ProjectID=${answer} 
     254    fi 
     255 
     256  fi # if ( ! ${x_p} ) 
     257 
     258  ProjectID=${ProjectID} 
     259  echo  ProjectID is ${ProjectID} at Jean-Zay 
     260 
    241261elif [ X"${SYSTEM}" == "Xada" ] ; then 
    242262  if ( ! ${x_m} ) ; then 
    … …  
    444464done 
    445465#- 
    446 # set default_project in libIGCM_sys_irene too. 
     466# set default_project in libIGCM_sys_irene and libIGCM_sys_jeanzay. 
    447467#- 
    448468if [ X"${SYSTEM}" == "Xirene" ] ; then 
    449469  i=${libIGCM}/libIGCM_sys/libIGCM_sys_irene.ksh 
     470  sed -i -e "s/::default_project::/${ProjectID}/" ${i} 
     471elif [ X"${SYSTEM}" == "Xjeanzay" ] ; then 
     472  i=${libIGCM}/libIGCM_sys/libIGCM_sys_jeanzay.ksh 
    450473  sed -i -e "s/::default_project::/${ProjectID}/" ${i} 
    451474fi 

  • trunk/libIGCM/libIGCM_sys/libIGCM_sys.ksh

    r1432 r1490  
    5050 
    5151#==================================================== 
     52 
    5253case $( hostname -s ) in 
    5354    ada*) 
    … …  
    9293        . ${libIGCM}/libIGCM_sys/libIGCM_sys_iitm.ksh;; 
    9394    *) 
    94         [ ! X${TaskType} = Xchecking ] && echo "Source machine dependent settings for a default machine." 
    95         CENTER=DEFAULT 
    96         SYSTEM=default 
    97         . ${libIGCM}/libIGCM_sys/libIGCM_sys_default.ksh;; 
     95        if [ "${SLURM_CLUSTER_NAME}" = "jean-zay" ] || [ "$( hostname -s |  cut -c "1-8"  )" = "jean-zay" ] ; then 
     96          [ ! X${TaskType} = Xchecking ] && echo "Source machine dependent settings for Jean-Zay Intel X-64." 
     97          CENTER=IDRIS 
     98          SYSTEM=jeanzay 
     99          . ${libIGCM}/libIGCM_sys/libIGCM_sys_jeanzay.ksh 
     100        else 
     101          [ ! X${TaskType} = Xchecking ] && echo "Source machine dependent settings for a default machine." 
     102          CENTER=DEFAULT 
     103          SYSTEM=default 
     104          . ${libIGCM}/libIGCM_sys/libIGCM_sys_default.ksh 
     105        fi;; 
    98106esac 
    99  
    100107# Set default umask (umask is 0027 on some machines : CCRT machine at least) 
    101108umask 0022 

  • trunk/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh

    r1489 r1490  
    1919 
    2020#D-#================================================== 
    21 #D-LibIGCM_sys for Irene 
     21#D-LibIGCM_sys for Jean-Zay 
    2222#D-#================================================== 
    2323#D- 
    … …  
    6868typeset LOGIN=${LOGIN:=$( whoami )} 
    6969# $hostname of the MASTER job 
    70 typeset MASTER=irene 
    71 # add default project on irene 
    72 typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} | cut -d@ -f1 ) 
    73 # jobWarningDelay in seconds 
    74 typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME} 
    75  
     70typeset MASTER=jeanzay 
     71# add default project set by ins_job 
     72typeset PROJECT=$( echo ${SLURM_JOB_ACCOUNT:=::default_project::} | cut -c "1-3" )  
     73### jobWarningDelay in seconds 
     74##typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME} 
     75  
    7676#D- 
    7777#D-#================================================== 
    … …  
    8080 
    8181# Submit command 
    82 typeset SUBMIT=${SUBMIT:=ccc_msub} 
     82typeset SUBMIT=${SUBMIT:=sbatch} 
    8383# rsync with path 
    8484typeset -r RSYNC=/usr/bin/rsync 
    … …  
    9090 
    9191#==================================================== 
     92# Access to module command 
     93#==================================================== 
     94. /usr/share/Modules/init/ksh 
     95 
     96#==================================================== 
    9297# Set environment tools (ferret, nco, cdo, rebuild, ...) 
    9398#==================================================== 
    … …  
    96101else 
    97102  module purge > /dev/null 2>&1 
    98   ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1 
    99   . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1 
    100   ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1 
    101   . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1 
    102  
    103   export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP 
     103  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay > /dev/null 2>&1 
     104#currently no atlas 
     105#  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_atlas_jeanzay > /dev/null 2>&1 
     106 
     107#  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP 
    104108  export UVCDAT_ANONYMOUS_LOG=FALSE 
    105109fi 
    … …  
    111115# For AddNoise and AddPertu3DOCE 
    112116##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/curie/bin 
    113 export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin 
     117#jg no addnoise 
     118#export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin 
    114119 
    115120# For rebuild 
    116121##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin 
    117 export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin 
     122#jg no yet any rebuild 
     123#export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin 
    118124 
    119125#==================================================== 
    120126# Host specific DIRECTORIES 
    121127#==================================================== 
    122  
    123 # ============ CESIUM START ============ # 
    124  
    125 #==================================================== 
    126 #- Mirror libIGCM from titane to cesium if needed 
    127 #ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' ) 
    128 #if [ ! ${ROOTSYS} = "home" ] ; then 
    129 #  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true} 
    130 #else 
    131 #  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false} 
    132 #fi 
    133  
    134 #==================================================== 
    135 #- libIGCM_POST 
    136 #if ( ${MirrorlibIGCM} ) ; then 
    137 #  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" ) 
    138 #  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM 
    139 #else 
    140 #  typeset -r libIGCM_POST=${libIGCM} 
    141 #fi 
    142  
    143 # ============ CESIUM  END  ============ # 
    144128 
    145129#==================================================== 
    … …  
    157141#==================================================== 
    158142#- SUBMIT_DIR : submission dir 
    159 typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}} 
     143typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${SLURM_SUBMIT_DIR}} 
    160144 
    161145#==================================================== 
    162146#- IN 
    163 typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM} 
     147typeset -r R_IN=${R_IN:=/gpfswork/rech/psl/commun/IGCM} 
    164148 
    165149#==================================================== 
    166150#- RUN_DIR_PATH : Temporary working directory (=> TMP) 
    167 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}} 
     151typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCH}/RUN_DIR/${SLURM_JOBID}_${$}} 
    168152 
    169153#==================================================== 
    … …  
    173157#==================================================== 
    174158#- HOST_MPIRUN_COMMAND 
    175 typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "} 
     159typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time srun --mpi=pmi2 "} 
    176160 
    177161#==================================================== 
    … …  
    180164 
    181165#==================================================== 
    182 #- set PackDefault to true on irene 
     166#- set PackDefault to true 
    183167PackDefault=true 
    184168 
    … …  
    203187#D-* Define STORAGE : Dedicated to small/medium files 
    204188#D-* Define R_OUT   : Output tree located on ARCHIVE 
    205 #D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files) 
    206 #D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes 
    207 #D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR 
     189#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monsitoring and atlas, and/or small files) 
     190#D-* Define R_BUF   : Output tree located on SCRATCH hosting files waiting for rebuild or pack processes 
     191#D-* if SpaceName=TEST everything is stored on SCRATCH 
    208192#D-* Examples: 
    209193#D- 
    … …  
    213197  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional) 
    214198  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = DEFAULT ]; then 
    215     # Default option: Change the dfldatadir according to the project used for submission 
    216     # The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT 
    217     if [ X"$( echo ${PROJECT} | grep cmip6 )" != "X" ] ; then 
    218       module switch dfldatadir dfldatadir/gencmip6 
    219       IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project" 
    220     else  
    221       module switch dfldatadir dfldatadir/${PROJECT} 
    222       IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}" 
    223     fi 
     199    # The variable DataProject is not set in in section UserChoices in config.card or it is set to DEFAULT 
     200    # Default option: Change IDRPROJ according to the project used for submission 
     201 
     202    # jg following line does not work in a submitted job: 
     203    eval $(idrenv -d ${PROJECT}) 
    224204  else 
    225205    # Use the project set by the variable DataProject in section UserChoices in config.card 
    226     module switch dfldatadir dfldatadir/${config_UserChoices_DataProject} 
    227     IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}" 
    228   fi 
    229  
    230  #==================================================== 
    231  #- RUN_DIR_PATH : Temporary working directory (=> TMP) 
    232  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}} 
     206    eval $(idrenv -d ${config_UserChoices_DataProject}) 
     207    IGCM_debug_Print 1 "Change DataProject red in config.card : ${config_UserChoices_DataProject}" 
     208  fi 
     209  IGCM_debug_Print 1 "Data project for output to be used: ${IDRPROJ}" 
     210 
     211  #==================================================== 
     212  #- RUN_DIR_PATH : Temporary working directory (=> TMP) 
     213  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCH}/RUN_DIR/${SLURM_JOBID}_${$}} 
    233214 
    234215  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then 
    … …  
    239220    #==================================================== 
    240221    #- ARCHIVE (dedicated to large files) 
    241     ARCHIVE=${CCCSTOREDIR} 
     222    ARCHIVE=${STORE} 
    242223  fi 
    243224 
    … …  
    249230    #==================================================== 
    250231    #- STORAGE (dedicated to small/medium files) 
    251     STORAGE=${CCCWORKDIR} 
     232    STORAGE=${WORK} 
    252233  fi 
    253234 
    … …  
    255236    #==================================================== 
    256237    #- R_OUT 
    257     R_OUT=${CCCSCRATCHDIR}/IGCM_OUT 
     238    R_OUT=${SCRATCH}/IGCM_OUT 
    258239 
    259240    #==================================================== 
    260241    #- R_FIG (hosting figures : monitoring and atlas, and/or small files) 
    261     R_FIG=${CCCSCRATCHDIR}/IGCM_OUT 
     242    R_FIG=${SCRATCH}/IGCM_OUT 
    262243 
    263244    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories" 
    … …  
    279260  #==================================================== 
    280261  #- R_BUF (ONLY FOR double copy an scratch) 
    281   R_BUF=${CCCSCRATCHDIR}/IGCM_OUT 
     262  R_BUF=${SCRATCH}/IGCM_OUT 
    282263 
    283264  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}" 
    … …  
    433414  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}" 
    434415 
    435   /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 
     416  /usr/bin/time sbatch ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 
    436417  status=$? 
    437418 
    … …  
    457438  fi 
    458439  typeset options status 
    459   options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out" 
    460  
    461   /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 
     440  options="-o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out" 
     441 
     442  sbatch ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 
    462443  status=$? 
    463444 
    … …  
    10891070      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used" 
    10901071      module purge > /dev/null 2>&1  
    1091       if [ X${compilerVersion} = XV17 ] ; then 
    1092           IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V17" 
    1093           ## EnvFile=/ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1 
    1094           EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1 
    1095       else 
    1096           IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}" 
    1097       fi 
     1072      EnvFile=/gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay 
    10981073  fi 
    10991074  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :"  
    … …  
    11011076  . ${EnvFile} 
    11021077 
    1103   IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene. " 
     1078  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Jean-Zay. " 
    11041079  module list 
    11051080 
    … …  
    15831558  fi 
    15841559 
    1585   if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then 
     1560  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORK} ] || [ X${RUN_DIR_PATH} = X${SCRATCH} ] || [ X${RUN_DIR_PATH} = X${STORE} ] ) ; then 
    15861561    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}" 
    15871562    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}" 
Note: See TracChangeset for help on using the changeset viewer.