Changeset 1502 for trunk/libIGCM


Ignore:
Timestamp:
11/20/19 11:32:17 (4 years ago)
Author:
acosce
Message:

clean all "ccc" write in the libIGCM_sys_jeanzay.

  • remove test quota on scratch because this command does not exist on Jean Zay
  • remove test to demigrate files on store because there is no archive on bands on Jean Zay
  • modify command to check JobId?, number of Job, list of Job
  • comment atlas part
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh

    r1500 r1502  
    102102  module purge > /dev/null 2>&1 
    103103  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay > /dev/null 2>&1 
    104 #currently no atlas 
    105 #  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_atlas_jeanzay > /dev/null 2>&1 
     104 
     105  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_atlas_jeanzay > /dev/null 2>&1 
    106106 
    107107#  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP 
    … …  
    347347  fi 
    348348  typeset IsArchivedFlag 
    349   IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 ) 
     349# To be implemented on Jean Zay  
     350#  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 ) 
    350351  IGCM_debug_PopStack "IGCM_sys_IsFileArchived" 
    351352 
    … …  
    538539    # 
    539540    # USUAL WAY 
    540     # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command : 
    541  #trouver la bonne commande sur jeanzay   ccc_hsm get -r $1 
     541    # add here command to demigrate all offline files to reduce time of this command  
     542    # actually on Jean Zay files are not archive on bands 
    542543 
    543544    i=0 
    … …  
    655656    if [ -f $2 ] ; then 
    656657      IGCM_debug_Print 1 "$2 already exist" 
    657  #trouver la bonne commande sur jeanzay     ccc_hsm get $2 
     658    # add here command to demigrate all offline files to reduce time of this command  
     659    # actually on Jean Zay files are not archive on bands 
     660 
    658661      exist=true 
    659662      if [ "X$( diff $1 $2 )" = X ] ; then 
    … …  
    773776    fi 
    774777    eval DEST=\${${#}} 
    775  #trouver la bonne commande sur jeanzay   ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 
     778    # add here command to demigrate all offline files to reduce time of this command  
     779    # actually on Jean Zay files are not archive on bands 
     780 
    776781    status=$? 
    777782    if [ ${status} -gt 0 ] ; then 
    … …  
    919924 
    920925    if [ "$#" -eq 1 ]; then 
    921       ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1 
    922       $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1 
     926#      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1 
    923927      status=$? 
    924928    else 
    925       ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1 
    926       $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1 
     929#      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1 
    927930      status=$? 
    928931    fi 
    … …  
    958961#    fi 
    959962 
    960     ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/bin/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1 
    961     $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1 
     963#    $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1 
     964 
    962965    status=$? 
    963966 
    … …  
    14261429function IGCM_sys_check_quota { 
    14271430  IGCM_debug_PushStack "IGCM_sys_check_quota" 
    1428   if ( $DEBUG_sys ) ; then 
    1429     echo "IGCM_sys_check_quota" 
    1430   fi 
    1431   # Limit of quota (in %) 
    1432   limit_quota=90 
    1433  
    1434   # Check of the volume 
    1435   volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}') 
    1436   volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}') 
    1437  
    1438   if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then 
    1439  
    1440     unit_avail=${volume_avail: -1} 
    1441     unit_quota=${volume_quota: -1} 
    1442  
    1443     if [ "${unit_quota}" = "*" ] ; then 
    1444       IGCM_debug_Print 1 "Please, check your quota of volume on scratch" 
    1445       IGCM_debug_Print 1 "More than 100% of your quota is used" 
    1446       IGCM_debug_Print 1 "Use the ccc_quota command to check" 
    1447       IGCM_debug_Print 1 "You must have more than 10% available to run" 
    1448       IGCM_debug_Exit "Not enough space to run ! STOP HERE" 
    1449       IGCM_debug_Verif_Exit 
    1450     fi 
    1451  
    1452     temp_avail=${volume_avail%%${volume_avail: -1}*} 
    1453     temp_quota=${volume_quota%%${volume_quota: -1}*} 
    1454  
    1455     if [ ! ${unit_avail} = ${unit_quota} ] ; then 
    1456  
    1457     # Convertion 
    1458       if [ ${volume_avail: -1} = "T" ] ; then 
    1459         (( temp_avail = temp_avail * 1000000000000 )) 
    1460       elif [ ${volume_avail: -1} = "G" ] ; then 
    1461         (( temp_avail = temp_avail * 1000000000 )) 
    1462       elif [ ${volume_avail: -1} = "M" ] ; then 
    1463         (( temp_avail = temp_avail * 1000000 )) 
    1464       elif [ ${volume_avail: -1} = "k" ] ; then 
    1465         (( temp_avail = temp_avail * 1000 )) 
    1466       else 
    1467         (( temp_avail = volume_avail )) 
    1468       fi 
    1469       if [ ${volume_quota: -1} = "T" ] ; then 
    1470         (( temp_quota = temp_quota * 1000000000000 )) 
    1471       elif [ ${volume_quota: -1} = "G" ] ; then 
    1472         (( temp_quota = temp_quota * 1000000000 )) 
    1473       elif [ ${volume_quota: -1} = "M" ] ; then 
    1474         (( temp_quota = temp_quota * 1000000 )) 
    1475       elif [ ${volume_quota: -1} = "k" ] ; then 
    1476         (( temp_quota = temp_quota * 1000 )) 
    1477       else 
    1478         (( temp_quota = volume_quota )) 
    1479       fi 
    1480     fi 
    1481  
    1482     quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc) 
    1483 #    echo "volume ratio is " $quota_volume 
    1484  
    1485     if [ ${quota_volume} -ge ${limit_quota} ] ; then 
    1486       IGCM_debug_Print 1 "Please, check your quota of volume on scratch" 
    1487       IGCM_debug_Print 1 "${quota_volume}% of your quota is used" 
    1488       IGCM_debug_Print 1 "Use the ccc_quota command to check" 
    1489       IGCM_debug_Print 1 "You must have more than 10% available to run" 
    1490       IGCM_debug_Exit "Not enough space to run ! STOP HERE" 
    1491       IGCM_debug_Verif_Exit 
    1492     fi 
    1493  
    1494   fi 
    1495  
    1496 # Check of the number of inodes 
    1497  
    1498   inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}') 
    1499   inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}') 
    1500  
    1501   if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then 
    1502  
    1503     unit_avail=${inode_avail: -1} 
    1504     unit_quota=${inode_quota: -1} 
    1505  
    1506     if [ "${unit_quota}" = "*" ] ; then 
    1507       IGCM_debug_Print 1 "Please, check your quota of inode on scratch" 
    1508       IGCM_debug_Print 1 "More than 100% of your quota is used" 
    1509       IGCM_debug_Print 1 "Use the ccc_quota command to check" 
    1510       IGCM_debug_Print 1 "You must have more than 10% available to run" 
    1511       IGCM_debug_Exit "Not enough space to run ! STOP HERE" 
    1512       IGCM_debug_Verif_Exit 
    1513     fi 
    1514  
    1515     temp_avail=${inode_avail%%${inode_avail: -1}*} 
    1516     temp_quota=${inode_quota%%${inode_quota: -1}*} 
    1517  
    1518     if [ ! ${unit_avail} = ${unit_quota} ] ; then 
    1519  
    1520     # Convertion 
    1521       if [ ${inode_avail: -1} = "T" ] ; then 
    1522         (( temp_avail = temp_avail * 1000000000000 )) 
    1523       elif [ ${inode_avail: -1} = "G" ] ; then 
    1524         (( temp_avail = temp_avail * 1000000000 )) 
    1525       elif [ ${inode_avail: -1} = "M" ] ; then 
    1526         (( temp_avail = temp_avail * 1000000 )) 
    1527       elif [ ${inode_avail: -1} = "k" ] ; then 
    1528         (( temp_avail = temp_avail * 1000 )) 
    1529       else 
    1530         (( temp_avail = inode_avail )) 
    1531       fi 
    1532  
    1533       if [ ${inode_quota: -1} = "T" ] ; then 
    1534         (( temp_quota = temp_quota * 1000000000000 )) 
    1535       elif [ ${inode_quota: -1} = "G" ] ; then 
    1536         (( temp_quota = temp_quota * 1000000000 )) 
    1537       elif [ ${inode_quota: -1} = "M" ] ; then 
    1538         (( temp_quota = temp_quota * 1000000 )) 
    1539       elif [ ${inode_quota: -1} = "k" ] ; then 
    1540         (( temp_quota = temp_quota * 1000 )) 
    1541       else 
    1542         (( temp_quota = inode_quota )) 
    1543       fi 
    1544     fi 
    1545     quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc) 
    1546 #    echo "inode ratio is " $quota_inode 
    1547  
    1548     if [ ${quota_inode} -ge ${limit_quota} ] ; then 
    1549       IGCM_debug_Print 1 "Please, check your quota of inode on scratch" 
    1550       IGCM_debug_Print 1 "${quota_inode}% of your quota is used" 
    1551       IGCM_debug_Print 1 "Use the ccc_quota command to check" 
    1552       IGCM_debug_Print 1 "You must have more than 10% available to run" 
    1553       IGCM_debug_Exit "Not enough space to run ! STOP HERE" 
    1554       IGCM_debug_Verif_Exit 
    1555     fi 
    1556   fi 
     1431  echo "IGCM_sys_check_quota is desactivated on JeanZay because there is actually no way to check scratch quota on this computing center (2019.11.19)" 
     1432#  if ( $DEBUG_sys ) ; then 
     1433#    echo "IGCM_sys_check_quota" 
     1434#  fi 
     1435#  # Limit of quota (in %) 
     1436#  limit_quota=90 
     1437# 
     1438#  # Check of the volume 
     1439#  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}') 
     1440#  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}') 
     1441# 
     1442#  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then 
     1443# 
     1444#    unit_avail=${volume_avail: -1} 
     1445#    unit_quota=${volume_quota: -1} 
     1446# 
     1447#    if [ "${unit_quota}" = "*" ] ; then 
     1448#      IGCM_debug_Print 1 "Please, check your quota of volume on scratch" 
     1449#      IGCM_debug_Print 1 "More than 100% of your quota is used" 
     1450#      IGCM_debug_Print 1 "Use the ccc_quota command to check" 
     1451#      IGCM_debug_Print 1 "You must have more than 10% available to run" 
     1452#      IGCM_debug_Exit "Not enough space to run ! STOP HERE" 
     1453#      IGCM_debug_Verif_Exit 
     1454#    fi 
     1455# 
     1456#    temp_avail=${volume_avail%%${volume_avail: -1}*} 
     1457#    temp_quota=${volume_quota%%${volume_quota: -1}*} 
     1458# 
     1459#    if [ ! ${unit_avail} = ${unit_quota} ] ; then 
     1460# 
     1461#    # Convertion 
     1462#      if [ ${volume_avail: -1} = "T" ] ; then 
     1463#        (( temp_avail = temp_avail * 1000000000000 )) 
     1464#      elif [ ${volume_avail: -1} = "G" ] ; then 
     1465#        (( temp_avail = temp_avail * 1000000000 )) 
     1466#      elif [ ${volume_avail: -1} = "M" ] ; then 
     1467#        (( temp_avail = temp_avail * 1000000 )) 
     1468#      elif [ ${volume_avail: -1} = "k" ] ; then 
     1469#        (( temp_avail = temp_avail * 1000 )) 
     1470#      else 
     1471#        (( temp_avail = volume_avail )) 
     1472#      fi 
     1473#      if [ ${volume_quota: -1} = "T" ] ; then 
     1474#        (( temp_quota = temp_quota * 1000000000000 )) 
     1475#      elif [ ${volume_quota: -1} = "G" ] ; then 
     1476#        (( temp_quota = temp_quota * 1000000000 )) 
     1477#      elif [ ${volume_quota: -1} = "M" ] ; then 
     1478#        (( temp_quota = temp_quota * 1000000 )) 
     1479#      elif [ ${volume_quota: -1} = "k" ] ; then 
     1480#        (( temp_quota = temp_quota * 1000 )) 
     1481#      else 
     1482#        (( temp_quota = volume_quota )) 
     1483#      fi 
     1484#    fi 
     1485# 
     1486#    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc) 
     1487##    echo "volume ratio is " $quota_volume 
     1488# 
     1489#    if [ ${quota_volume} -ge ${limit_quota} ] ; then 
     1490#      IGCM_debug_Print 1 "Please, check your quota of volume on scratch" 
     1491#      IGCM_debug_Print 1 "${quota_volume}% of your quota is used" 
     1492#      IGCM_debug_Print 1 "Use the ccc_quota command to check" 
     1493#      IGCM_debug_Print 1 "You must have more than 10% available to run" 
     1494#      IGCM_debug_Exit "Not enough space to run ! STOP HERE" 
     1495#      IGCM_debug_Verif_Exit 
     1496#    fi 
     1497# 
     1498#  fi 
     1499# 
     1500## Check of the number of inodes 
     1501# 
     1502#  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}') 
     1503#  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}') 
     1504# 
     1505#  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then 
     1506# 
     1507#    unit_avail=${inode_avail: -1} 
     1508#    unit_quota=${inode_quota: -1} 
     1509# 
     1510#    if [ "${unit_quota}" = "*" ] ; then 
     1511#      IGCM_debug_Print 1 "Please, check your quota of inode on scratch" 
     1512#      IGCM_debug_Print 1 "More than 100% of your quota is used" 
     1513#      IGCM_debug_Print 1 "Use the ccc_quota command to check" 
     1514#      IGCM_debug_Print 1 "You must have more than 10% available to run" 
     1515#      IGCM_debug_Exit "Not enough space to run ! STOP HERE" 
     1516#      IGCM_debug_Verif_Exit 
     1517#    fi 
     1518# 
     1519#    temp_avail=${inode_avail%%${inode_avail: -1}*} 
     1520#    temp_quota=${inode_quota%%${inode_quota: -1}*} 
     1521# 
     1522#    if [ ! ${unit_avail} = ${unit_quota} ] ; then 
     1523# 
     1524#    # Convertion 
     1525#      if [ ${inode_avail: -1} = "T" ] ; then 
     1526#        (( temp_avail = temp_avail * 1000000000000 )) 
     1527#      elif [ ${inode_avail: -1} = "G" ] ; then 
     1528#        (( temp_avail = temp_avail * 1000000000 )) 
     1529#      elif [ ${inode_avail: -1} = "M" ] ; then 
     1530#        (( temp_avail = temp_avail * 1000000 )) 
     1531#      elif [ ${inode_avail: -1} = "k" ] ; then 
     1532#        (( temp_avail = temp_avail * 1000 )) 
     1533#      else 
     1534#        (( temp_avail = inode_avail )) 
     1535#      fi 
     1536# 
     1537#      if [ ${inode_quota: -1} = "T" ] ; then 
     1538#        (( temp_quota = temp_quota * 1000000000000 )) 
     1539#      elif [ ${inode_quota: -1} = "G" ] ; then 
     1540#        (( temp_quota = temp_quota * 1000000000 )) 
     1541#      elif [ ${inode_quota: -1} = "M" ] ; then 
     1542#        (( temp_quota = temp_quota * 1000000 )) 
     1543#      elif [ ${inode_quota: -1} = "k" ] ; then 
     1544#        (( temp_quota = temp_quota * 1000 )) 
     1545#      else 
     1546#        (( temp_quota = inode_quota )) 
     1547#      fi 
     1548#    fi 
     1549#    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc) 
     1550##    echo "inode ratio is " $quota_inode 
     1551# 
     1552#    if [ ${quota_inode} -ge ${limit_quota} ] ; then 
     1553#      IGCM_debug_Print 1 "Please, check your quota of inode on scratch" 
     1554#      IGCM_debug_Print 1 "${quota_inode}% of your quota is used" 
     1555#      IGCM_debug_Print 1 "Use the ccc_quota command to check" 
     1556#      IGCM_debug_Print 1 "You must have more than 10% available to run" 
     1557#      IGCM_debug_Exit "Not enough space to run ! STOP HERE" 
     1558#      IGCM_debug_Verif_Exit 
     1559#    fi 
     1560#  fi 
    15571561  IGCM_debug_PopStack "IGCM_sys_check_quota" 
    15581562} 
    … …  
    15681572    echo "IGCM_sys_check_quota" 
    15691573  fi 
    1570  
    1571   ssh irene191 /usr/bin/ccc_myproject > $1 
     1574#need to implemented on Jean Zay  
     1575  touch $1 
     1576#  ssh irene191 /usr/bin/ccc_myproject > $1 
    15721577 
    15731578  IGCM_debug_PopStack "IGCM_sys_projectAccounting" 
    … …  
    16011606  fi 
    16021607 
    1603   # With -f option, the full job name is given in the last column 
    1604   ID=$( ccc_mstat -f -u $2 | \ 
    1605         gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' ) 
     1608  ID=$( squeue -u $user | grep Job_${config_UserChoices_JobName} | gawk ' { print $1}' ) 
     1609 
    16061610 
    16071611  eval ${3}=${ID} 
    … …  
    16211625 
    16221626  # With -f option, the full job name is given in the last column 
    1623   NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' ) 
     1627  NbRun=$(squeue -u $user | \  
     1628      grep -v JOBID | gawk 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' ) 
    16241629 
    16251630  eval ${2}=${NbRun} 
    … …  
    16391644  fi 
    16401645 
    1641   # With -f option, the full job name is given in the last column 
    1642   set -A JobList $( ccc_mstat -f | gawk -v User=$1             \ 
    1643                                         '( $2  == User      && \ 
    1644                                            $NF != /TS/      && \ 
    1645                                            $NF !~ /PACK/    && \ 
    1646                                            $NF !~ /REBUILD/ && \ 
    1647                                            $NF !~ /pack/ )     \ 
    1648                                          { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" ) 
    1649  
     1646# to be implemented on Jean Zay  
     1647 set -A JobList $( squeue -u $user | gawk '{print $3}' | grep -v NAME) 
     1648#  # With -f option, the full job name is given in the last column 
     1649#  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \ 
     1650#                                        '( $2  == User      && \ 
     1651#                                           $NF != /TS/      && \ 
     1652#                                           $NF !~ /PACK/    && \ 
     1653#                                           $NF !~ /REBUILD/ && \ 
     1654#                                           $NF !~ /pack/ )     \ 
     1655#                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" ) 
     1656# 
    16501657  eval set -A ${2} ${JobList[*]} 
    16511658 
    … …  
    16651672 
    16661673  typeset status 
    1667  
    1668   \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 
    1669   status=$? 
    1670   if [ ${status} -gt 0 ] ; then 
    1671     echo "IGCM_sys_atlas : error code ${status}" 
    1672     cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 
    1673     \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 
    1674     IGCM_debug_PopStack "IGCM_sys_atlas" 
    1675     return 1 
    1676   else 
    1677     IGCM_debug_PopStack "IGCM_sys_atlas" 
    1678     return 0 
    1679   fi 
    1680  
     1674# To be implemented on Jean Zay  
     1675# 
     1676#  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 
     1677#  status=$? 
     1678#  if [ ${status} -gt 0 ] ; then 
     1679#    echo "IGCM_sys_atlas : error code ${status}" 
     1680#    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 
     1681#    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 
     1682#    IGCM_debug_PopStack "IGCM_sys_atlas" 
     1683#    return 1 
     1684#  else 
     1685#    IGCM_debug_PopStack "IGCM_sys_atlas" 
     1686#    return 0 
     1687#  fi 
     1688# 
    16811689  IGCM_debug_PopStack "IGCM_sys_atlas" 
    16821690} 
Note: See TracChangeset for help on using the changeset viewer.