Ignore:
Timestamp:
04/14/15 17:00:21 (9 years ago)
Author:
sdipsl
Message:
  • Extend Big Brother scope to also watch post-processing jobs. Only the metrics LMDZ job for now. See #252
  • Some refactoring
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh

    r1162 r1189  
    171171  fi 
    172172 
    173   if [ X${FlushAMQP} = XTRUE ] ; then 
     173  if ( ${FlushAMQP} ) ; then 
    174174    mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient}  < ${buffer} # send buffer 
    175175    rm -f ${buffer}                                                    # cleaning behind us 
    … …  
    325325    fi 
    326326 
    327     # Unplugged message 2000 handling for now. To ease downstream treatment. 
     327    # Unplugged message 4000 handling for now. To ease downstream treatment. 
    328328    #if [ X${ActivateBigBro} = Xtrue ] ; then 
    329329    #  # RabbitMQ message code "PUSHSTACK" 
    330     #  code=2000 
     330    #  code=4000 
    331331    #  # RabbitMQ message body 
    332332    #  Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${IGCM_debug_Stack[0]}\",\"arguments\":[\"${IGCM_debug_StackArgs[0]}\"],\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    … …  
    530530      if [ X${ActivateBigBro} = Xtrue ] ; then 
    531531        # RabbitMQ message code "ERROR HAS BEEN TRIGGERED" 
    532         code=9000 
     532        code=4900 
    533533        # RabbitMQ message body 
    534534        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    … …  
    542542      fi 
    543543 
    544       # Unplugged message 3000 handling for now. To ease downstream treatment. 
     544      # Unplugged message 4100 handling for now. To ease downstream treatment. 
    545545      #if [ X${ActivateBigBro} = Xtrue ] ; then 
    546546      #  # RabbitMQ message code "POPSTACK" 
    547       #  code=3000 
     547      #  code=4100 
    548548      #  # RabbitMQ message body 
    549549      #  Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    … …  
    570570 
    571571#D-#================================================================== 
    572 #D-function IGCM_debug_ActivateBigBro 
     572#D-function IGCM_debug_BigBro_Initialize 
    573573#D-* Purpose: switch rabbitMQ on 
    574574#D- 
    575 function IGCM_debug_ActivateBigBro { 
    576   IGCM_debug_PushStack "IGCM_debug_ActivateBigBro" 
     575function IGCM_debug_BigBro_Initialize { 
     576  IGCM_debug_PushStack "IGCM_debug_BigBro_Initialize" 
    577577 
    578578# Message type standard fields: 
    … …  
    586586    jobuid=$(uuidgen) 
    587587 
    588     if ( ${FirstInitialize} ) ; then 
    589       # RabbitMQ message code "BEGIN A SIMULATION" 
    590       code=0000 
    591       # create and persist a unique id for this simulation 
    592       simuid=$(uuidgen) 
    593       IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid} 
    594       # Standard fields for the first message 
    595       genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) 
    596       # RabbitMQ message body with specific fields associated message codes treated here 
    597       Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    598       # Fill the rabbitMQ queue (the config.card in use will be sent) 
    599       IGCM_debug_sendAMQP activate 
    600     else 
    601       # RabbitMQ message code "A NEW JOB IS RUNNING PART OF A SIMULATION" 
    602       code=1000 
     588    if [ X${TaskType} = Xcomputing ]; then 
     589      if ( ${FirstInitialize} ) ; then 
     590        # RabbitMQ message code "BEGIN A SIMULATION" 
     591        code=0000 
     592        # create and persist a unique id for this simulation 
     593        simuid=$(uuidgen) 
     594        IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid} 
     595        # Standard fields for the first message 
     596        genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) 
     597        # RabbitMQ message body with specific fields associated message codes treated here 
     598        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
     599        # Fill the rabbitMQ queue (the config.card in use will be sent) 
     600        IGCM_debug_sendAMQP activate 
     601      else 
     602        # RabbitMQ message code "A NEW COMPUTING JOB IS RUNNING PART OF A SIMULATION" 
     603        code=1000 
     604        # retrieve this simulation's unique id 
     605        IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid 
     606        simuid=${run_Configuration_simuid} 
     607        # Using standard fields for message others than the first one. Still subject to change 
     608        genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) 
     609        # RabbitMQ message body with specific fields associated message codes treated here 
     610        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
     611        # Fill the rabbitMQ queue 
     612        IGCM_debug_sendAMQP 
     613      fi 
     614 
     615      # NOT VERY NICE BUT ... IT WORKS 
     616      # Be sure that the genericSimulationID will be small from now on 
     617      # Using standard fields for messages others than the first one. Still subject to change 
     618      genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) 
     619 
     620    elif [ X${TaskType} = Xpost-processing ]; then 
     621      # RabbitMQ message code "A NEW POST-PROCESSING JOB IS RUNNING PART OF A SIMULATION" 
     622      code=2000 
    603623      # retrieve this simulation's unique id 
    604624      IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid 
    … …  
    611631      IGCM_debug_sendAMQP 
    612632    fi 
    613  
    614     # NOT VERY NICE BUT ... IT WORKS 
    615     # Be sure that the genericSimulationID will be small from now on 
    616     # Using standard fields for messages others than the first one. Still subject to change 
    617     genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) 
    618  
    619633    # Turn the flag on 
    620634    ActivateBigBro=true 
    621   fi 
    622   IGCM_debug_PopStack "IGCM_debug_ActivateBigBro" 
     635    # Dont flush by default 
     636    FlushAMQP=false 
     637  fi 
     638  IGCM_debug_PopStack "IGCM_debug_BigBro_Initialize" 
     639} 
     640 
     641#D-#================================================================== 
     642#D-function IGCM_debug_BigBro_Finalize 
     643#D-* Purpose: Finalize rabbitMQ messages exchanges 
     644#D- 
     645function IGCM_debug_BigBro_Finalize { 
     646  IGCM_debug_PushStack "IGCM_debug_BigBro_Finalize" 
     647 
     648  # Message type standard fields: 
     649  # https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields 
     650 
     651  # Message type dictionnary and custom fields: 
     652  # https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary 
     653 
     654  if ( $DEBUG_debug ) ; then 
     655    if [ X${ActivateBigBro} = Xtrue ] ; then 
     656      if [ X${TaskType} = Xcomputing ]; then 
     657        if ( ${simulationIsOver} ) ; then 
     658          # RabbitMQ message code "SIMULATION ENDS" 
     659          code=0100 
     660          FlushAMQP=true 
     661        else 
     662          # RabbitMQ message code "COMPUTING JOB ENDS" 
     663          code=1100 
     664        fi 
     665      elif [ X${TaskType} = Xpost-processing ]; then 
     666        # RabbitMQ message code "POST-PROCESSING JOB ENDS" 
     667        code=2100 
     668        FlushAMQP=true 
     669      fi 
     670      # RabbitMQ message body 
     671      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
     672      # Fill the rabbitMQ queue 
     673      IGCM_debug_sendAMQP 
     674    fi 
     675  fi 
     676   
     677  IGCM_debug_PopStack "IGCM_debug_BigBro_Finalize" 
    623678} 
    624679 
    … …  
    663718      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) 
    664719      # To be sure we flush message buffer and clean up behind us before the end of the simulation 
    665       FlushAMQP=TRUE 
     720      FlushAMQP=true 
    666721      # Fill the rabbitMQ queue 
    667722      IGCM_debug_sendAMQP 
Note: See TracChangeset for help on using the changeset viewer.