Changeset 1189 for trunk


Ignore:
Timestamp:
04/14/15 17:00:21 (9 years ago)
Author:
sdipsl
Message:
  • Extend Big Brother scope to also watch post-processing jobs. Only the metrics LMDZ job for now. See #252
  • Some refactoring
Location:
trunk/libIGCM
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/libIGCM/AA_job

    r1184 r1189  
    241241# Activate BigBrother so as to supervise this simulation 
    242242# ------------------------------------------------------------------ 
    243 IGCM_debug_ActivateBigBro 
     243IGCM_debug_BigBro_Initialize 
    244244 
    245245#D- 

  • trunk/libIGCM/AA_metrics_LMDZ

    r1184 r1189  
    118118IGCM_config_CommonConfiguration ${CARD_DIR}/config.card 
    119119 
     120# ------------------------------------------------------------------ 
     121# Activate BigBrother so as to supervise this job 
     122# ------------------------------------------------------------------ 
     123IGCM_debug_BigBro_Initialize 
     124 
    120125#================================== 
    121126# Then set up several directories 
    … …  
    181186# Clean RUN_DIR_PATH (necessary for cesium and titane only) 
    182187IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH} 
     188 
     189# ------------------------------------------------------------------ 
     190# Finalize BigBrother to inform that the jobs end 
     191# ------------------------------------------------------------------ 
     192IGCM_debug_BigBro_Finalize 

  • trunk/libIGCM/libIGCM_config/libIGCM_config.ksh

    r1173 r1189  
    12481248    # End of entire simulation # 
    12491249    #==========================# 
     1250    simulationIsOver=true 
    12501251 
    12511252    # Mail notification 
    … …  
    12561257    IGCM_debug_Print 1 "Normal End of computation." 
    12571258 
    1258     if ( $DEBUG_debug ) ; then 
    1259       if [ X${ActivateBigBro} = Xtrue ] ; then 
    1260         # RabbitMQ message code "END OF SIMULATION" 
    1261         code=0100 
    1262         # RabbitMQ message body 
    1263         Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    1264         # Fill the rabbitMQ queue 
    1265         IGCM_debug_sendAMQP 
    1266       fi 
    1267     fi 
    1268  
    1269     # To be sure we flush message buffer and clean up behind us before the end of the simulation 
    1270     FlushAMQP=TRUE 
    1271  
    12721259  else 
    12731260    #=================# 
    12741261    # Submit next job # 
    12751262    #=================# 
     1263    simulationIsOver=false 
    12761264 
    12771265    IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "OnQueue" 
    … …  
    12911279  fi 
    12921280 
    1293   # Supress Non Deleted Boundary files 
    1294   #if [ ${DRYRUN} -le 2 ] ; then 
    1295   #  IGCM_comp_DelFixeBoundaryFiles 
    1296   #  ls -la 
    1297   #fi 
    1298  
    12991281  # Clean ${RUN_DIR}=${RUN_DIR_PATH}/${config_UserChoices_JobName}.${$} 
    13001282  # Only for production run (No clean up in DEV or DEB mode) 
    … …  
    13031285  [ X${JobType} = XRUN ] && IGCM_sys_RmRunDir -rf ${RUN_DIR_PATH} 
    13041286 
    1305   if ( $DEBUG_debug ) ; then 
    1306     # Inform the rabbitMQ queue 
    1307     if [ X${ActivateBigBro} = Xtrue ] ; then 
    1308       # RabbitMQ message code "WAITING" 
    1309       code=1100 
    1310       # RabbitMQ message body 
    1311       Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    1312       # Fill the rabbitMQ queue 
    1313       IGCM_debug_sendAMQP 
    1314     fi 
    1315   fi 
     1287  # Inform the rabbitMQ queue 
     1288  IGCM_debug_BigBro_Finalize 
    13161289 
    13171290  IGCM_debug_PopStack "IGCM_config_Finalize" 

  • trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh

    r1162 r1189  
    171171  fi 
    172172 
    173   if [ X${FlushAMQP} = XTRUE ] ; then 
     173  if ( ${FlushAMQP} ) ; then 
    174174    mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient}  < ${buffer} # send buffer 
    175175    rm -f ${buffer}                                                    # cleaning behind us 
    … …  
    325325    fi 
    326326 
    327     # Unplugged message 2000 handling for now. To ease downstream treatment. 
     327    # Unplugged message 4000 handling for now. To ease downstream treatment. 
    328328    #if [ X${ActivateBigBro} = Xtrue ] ; then 
    329329    #  # RabbitMQ message code "PUSHSTACK" 
    330     #  code=2000 
     330    #  code=4000 
    331331    #  # RabbitMQ message body 
    332332    #  Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${IGCM_debug_Stack[0]}\",\"arguments\":[\"${IGCM_debug_StackArgs[0]}\"],\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    … …  
    530530      if [ X${ActivateBigBro} = Xtrue ] ; then 
    531531        # RabbitMQ message code "ERROR HAS BEEN TRIGGERED" 
    532         code=9000 
     532        code=4900 
    533533        # RabbitMQ message body 
    534534        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    … …  
    542542      fi 
    543543 
    544       # Unplugged message 3000 handling for now. To ease downstream treatment. 
     544      # Unplugged message 4100 handling for now. To ease downstream treatment. 
    545545      #if [ X${ActivateBigBro} = Xtrue ] ; then 
    546546      #  # RabbitMQ message code "POPSTACK" 
    547       #  code=3000 
     547      #  code=4100 
    548548      #  # RabbitMQ message body 
    549549      #  Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    … …  
    570570 
    571571#D-#================================================================== 
    572 #D-function IGCM_debug_ActivateBigBro 
     572#D-function IGCM_debug_BigBro_Initialize 
    573573#D-* Purpose: switch rabbitMQ on 
    574574#D- 
    575 function IGCM_debug_ActivateBigBro { 
    576   IGCM_debug_PushStack "IGCM_debug_ActivateBigBro" 
     575function IGCM_debug_BigBro_Initialize { 
     576  IGCM_debug_PushStack "IGCM_debug_BigBro_Initialize" 
    577577 
    578578# Message type standard fields: 
    … …  
    586586    jobuid=$(uuidgen) 
    587587 
    588     if ( ${FirstInitialize} ) ; then 
    589       # RabbitMQ message code "BEGIN A SIMULATION" 
    590       code=0000 
    591       # create and persist a unique id for this simulation 
    592       simuid=$(uuidgen) 
    593       IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid} 
    594       # Standard fields for the first message 
    595       genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) 
    596       # RabbitMQ message body with specific fields associated message codes treated here 
    597       Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    598       # Fill the rabbitMQ queue (the config.card in use will be sent) 
    599       IGCM_debug_sendAMQP activate 
    600     else 
    601       # RabbitMQ message code "A NEW JOB IS RUNNING PART OF A SIMULATION" 
    602       code=1000 
     588    if [ X${TaskType} = Xcomputing ]; then 
     589      if ( ${FirstInitialize} ) ; then 
     590        # RabbitMQ message code "BEGIN A SIMULATION" 
     591        code=0000 
     592        # create and persist a unique id for this simulation 
     593        simuid=$(uuidgen) 
     594        IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid} 
     595        # Standard fields for the first message 
     596        genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) 
     597        # RabbitMQ message body with specific fields associated message codes treated here 
     598        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
     599        # Fill the rabbitMQ queue (the config.card in use will be sent) 
     600        IGCM_debug_sendAMQP activate 
     601      else 
     602        # RabbitMQ message code "A NEW COMPUTING JOB IS RUNNING PART OF A SIMULATION" 
     603        code=1000 
     604        # retrieve this simulation's unique id 
     605        IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid 
     606        simuid=${run_Configuration_simuid} 
     607        # Using standard fields for message others than the first one. Still subject to change 
     608        genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) 
     609        # RabbitMQ message body with specific fields associated message codes treated here 
     610        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
     611        # Fill the rabbitMQ queue 
     612        IGCM_debug_sendAMQP 
     613      fi 
     614 
     615      # NOT VERY NICE BUT ... IT WORKS 
     616      # Be sure that the genericSimulationID will be small from now on 
     617      # Using standard fields for messages others than the first one. Still subject to change 
     618      genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) 
     619 
     620    elif [ X${TaskType} = Xpost-processing ]; then 
     621      # RabbitMQ message code "A NEW POST-PROCESSING JOB IS RUNNING PART OF A SIMULATION" 
     622      code=2000 
    603623      # retrieve this simulation's unique id 
    604624      IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid 
    … …  
    611631      IGCM_debug_sendAMQP 
    612632    fi 
    613  
    614     # NOT VERY NICE BUT ... IT WORKS 
    615     # Be sure that the genericSimulationID will be small from now on 
    616     # Using standard fields for messages others than the first one. Still subject to change 
    617     genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) 
    618  
    619633    # Turn the flag on 
    620634    ActivateBigBro=true 
    621   fi 
    622   IGCM_debug_PopStack "IGCM_debug_ActivateBigBro" 
     635    # Dont flush by default 
     636    FlushAMQP=false 
     637  fi 
     638  IGCM_debug_PopStack "IGCM_debug_BigBro_Initialize" 
     639} 
     640 
     641#D-#================================================================== 
     642#D-function IGCM_debug_BigBro_Finalize 
     643#D-* Purpose: Finalize rabbitMQ messages exchanges 
     644#D- 
     645function IGCM_debug_BigBro_Finalize { 
     646  IGCM_debug_PushStack "IGCM_debug_BigBro_Finalize" 
     647 
     648  # Message type standard fields: 
     649  # https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields 
     650 
     651  # Message type dictionnary and custom fields: 
     652  # https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary 
     653 
     654  if ( $DEBUG_debug ) ; then 
     655    if [ X${ActivateBigBro} = Xtrue ] ; then 
     656      if [ X${TaskType} = Xcomputing ]; then 
     657        if ( ${simulationIsOver} ) ; then 
     658          # RabbitMQ message code "SIMULATION ENDS" 
     659          code=0100 
     660          FlushAMQP=true 
     661        else 
     662          # RabbitMQ message code "COMPUTING JOB ENDS" 
     663          code=1100 
     664        fi 
     665      elif [ X${TaskType} = Xpost-processing ]; then 
     666        # RabbitMQ message code "POST-PROCESSING JOB ENDS" 
     667        code=2100 
     668        FlushAMQP=true 
     669      fi 
     670      # RabbitMQ message body 
     671      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
     672      # Fill the rabbitMQ queue 
     673      IGCM_debug_sendAMQP 
     674    fi 
     675  fi 
     676   
     677  IGCM_debug_PopStack "IGCM_debug_BigBro_Finalize" 
    623678} 
    624679 
    … …  
    663718      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    664719      # To be sure we flush message buffer and clean up behind us before the end of the simulation 
    665       FlushAMQP=TRUE 
     720      FlushAMQP=true 
    666721      # Fill the rabbitMQ queue 
    667722      IGCM_debug_sendAMQP 
Note: See TracChangeset for help on using the changeset viewer.