[4237] | 1 | #!/bin/bash |
---|
| 2 | #! |
---|
| 3 | #BSUB -q poe_short |
---|
[4373] | 4 | #BSUB -n TOTAL_NPROCS |
---|
[4237] | 5 | #BSUB -a poe |
---|
| 6 | #BSUB -J MPI_config |
---|
| 7 | #BSUB -o poe.stdout.%J |
---|
| 8 | #BSUB -e poe.stderr.%J |
---|
| 9 | |
---|
| 10 | |
---|
| 11 | |
---|
| 12 | ############################################################### |
---|
| 13 | # Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these |
---|
| 14 | # (via sed operating on this template job file). |
---|
| 15 | # |
---|
| 16 | OCEANCORES=NPROCS |
---|
[4373] | 17 | XIOS_NUMPROCS=NXIOPROCS |
---|
[4237] | 18 | export SETTE_DIR=DEF_SETTE_DIR |
---|
| 19 | |
---|
| 20 | ############################################################### |
---|
| 21 | # |
---|
| 22 | # set up mpp computing environment |
---|
| 23 | # |
---|
| 24 | # Local settings for machine IBM Power6 (VARGAS at IDRIS France) |
---|
| 25 | # |
---|
| 26 | |
---|
| 27 | # export MPIRUN="mpirun -n $OCEANCORES" |
---|
| 28 | |
---|
| 29 | export MPIRUN="mpirun.lsf" |
---|
[4373] | 30 | export LANG=en_US |
---|
| 31 | export NLSPATH=/opt/ibmhpc/pecurrent/base/msg/%L/%N:$NLSPATH |
---|
| 32 | export MP_LABELIO=yes |
---|
| 33 | export MP_STDOUTMODE=unordered |
---|
| 34 | export MP_EUILIB=us |
---|
| 35 | export MP_INFOLEVEL=0 |
---|
| 36 | export MP_EUIDEVELOP=min |
---|
| 37 | export MP_INSTANCES=1 |
---|
| 38 | export MP_SHARED_MEMORY=yes |
---|
| 39 | export MP_EAGER_LIMIT_LOCAL=2000000000 |
---|
| 40 | export MP_EUIDEVICE=sn_single |
---|
| 41 | export MP_BULK_MIN_MSG_SIZE=16384 |
---|
| 42 | export MP_POLLING_INTERVAL=1000000 |
---|
| 43 | export MP_DEVTYPE=ib |
---|
| 44 | export MP_EAGER_LIMIT=262144 |
---|
| 45 | export MP_TASK_AFFINITY=core |
---|
| 46 | export MP_BINDPROC=yes |
---|
| 47 | export MP_DEBUG_CHECK_PARAM=no |
---|
| 48 | export MP_MSG_API=mpi |
---|
| 49 | export MP_USE_BULK_XFER=yes |
---|
| 50 | export MP_WAIT_MODE=poll |
---|
| 51 | export LDR_CNTRL=TEXTPSIZE=64K@STACKPSIZE=64K@DATAPSIZE=64K |
---|
| 52 | export LD_LIBRARY_PATH=/users/home/ans040/local/lib:${LD_LIBRARY_PATH} |
---|
[4237] | 53 | |
---|
[4373] | 54 | XIOS_SERVER_PATHNAME="/users/home/ans040/SOFTWARE/XIOS/trunk/bin/xios_server.exe" |
---|
| 55 | |
---|
| 56 | |
---|
[4237] | 57 | # |
---|
| 58 | # load sette functions (only post_test_tidyup needed) |
---|
| 59 | # |
---|
| 60 | . ${SETTE_DIR}/all_functions.sh |
---|
| 61 | |
---|
| 62 | |
---|
| 63 | # Don't remove neither change the following line |
---|
| 64 | # BODY |
---|
| 65 | |
---|
| 66 | # |
---|
| 67 | # These variables are needed by post_test_tidyup function in all_functions.sh |
---|
| 68 | # |
---|
| 69 | export EXE_DIR=DEF_EXE_DIR |
---|
| 70 | export INPUT_DIR=DEF_INPUT_DIR |
---|
| 71 | export CONFIG_DIR=DEF_CONFIG_DIR |
---|
| 72 | export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION |
---|
| 73 | export NEW_CONF=DEF_NEW_CONF |
---|
| 74 | export CMP_NAM=DEF_CMP_NAM |
---|
| 75 | export TEST_NAME=DEF_TEST_NAME |
---|
| 76 | # |
---|
| 77 | # end of set up |
---|
| 78 | |
---|
| 79 | |
---|
| 80 | ############################################################### |
---|
| 81 | # |
---|
| 82 | # change to the working directory |
---|
| 83 | # |
---|
| 84 | cd ${EXE_DIR} |
---|
| 85 | |
---|
| 86 | # |
---|
| 87 | echo Running on host `hostname` |
---|
| 88 | echo Time is `date` |
---|
| 89 | echo Directory is `pwd` |
---|
| 90 | # |
---|
| 91 | # Run the parallel MPI executable |
---|
| 92 | # |
---|
| 93 | # echo "Running time ${MPIRUN} " $OCEANCORES " ./opa" |
---|
| 94 | # |
---|
| 95 | |
---|
| 96 | if [ MPI_FLAG == "yes" ]; then |
---|
[4373] | 97 | if [ $XIOS_NUMPROCS -eq 0 ]; then |
---|
| 98 | time ${MPIRUN} ./opa |
---|
| 99 | else |
---|
| 100 | |
---|
| 101 | export MP_PGMMODEL=mpmd |
---|
| 102 | export MP_CMDFILE=xioscmdfile |
---|
| 103 | rm xioscmdfile &> /dev/null |
---|
| 104 | touch xioscmdfile |
---|
| 105 | k=0 |
---|
| 106 | while (( k < $OCEANCORES )) ; do |
---|
| 107 | echo ./opa >> xioscmdfile |
---|
| 108 | k=`expr $k + 1` |
---|
| 109 | done |
---|
| 110 | k=0 |
---|
| 111 | while (( k < $XIOS_NUMPROCS )) ; do |
---|
| 112 | echo $XIOS_SERVER_PATHNAME >> xioscmdfile |
---|
| 113 | k=`expr $k + 1` |
---|
| 114 | done |
---|
| 115 | |
---|
| 116 | time ${MPIRUN} |
---|
| 117 | fi |
---|
[4237] | 118 | else |
---|
| 119 | time ./opa |
---|
| 120 | fi |
---|
| 121 | # |
---|
| 122 | post_test_tidyup |
---|
| 123 | |
---|
| 124 | # END_BODY |
---|
| 125 | # Don't remove neither change the previous line |
---|
| 126 | |
---|
| 127 | exit |
---|