- Timestamp:
- 02/06/15 17:14:55 (10 years ago)
- Location:
- TOOLS/ConsoGENCMIP6
- Files:
-
- 1 added
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
TOOLS/ConsoGENCMIP6/bin/conso_gencmip6.py
r2411 r2412 1 #!/bin/bash 2 # 3 # cron pour garder trace de la compta et des volumes de fichiers crees 4 # a executer chaque matin a 6h00 5 # ====================================================================== 6 #. /etc/profile 7 8 9 # Functions 10 # ========= 11 function get_gencmip6 { 12 ccc_myproject | sed -e'1,/gencmip6/d' | grep -m 1 $1 13 } 14 15 function get_gencmip6_login { 16 ccc_myproject | sed -n -e'/gencmip6/,/^$/p' | head -n -1 | tail -n +3 17 } 18 19 20 # Default values 21 # ============== 22 fg_dry=false 23 fg_verbose=false 24 25 fg_all=true 26 fg_login=true 27 fg_bilan=true 28 fg_store=true 29 30 # Get arguments from command line 31 # =============================== 32 while getopts :hdaltsv Opt ; do 33 case $Opt in 34 h) 35 echo "usage: $0 [-h] [-a] [-l] [-b] [-s] [-d] [-v]" 36 echo "" 37 echo "options :" 38 echo " -h : print this help and exit" 39 echo " -d : dry run, no file produced" 40 echo " -a : produce all files (default)" 41 echo " -l : produce login file" 42 echo " -b : produce bilan file" 43 echo " -s : produce store file" 44 echo " -v : verbose" 45 exit 0 ;; 46 d) 47 fg_dry=true 48 ;; 49 a) 50 fg_all=true 51 ;; 52 l) 53 fg_login=true 54 fg_all=false 55 fg_bilan=false 56 fg_store=false 57 ;; 58 t) 59 fg_bilan=true 60 fg_all=false 61 fg_login=false 62 fg_store=false 63 ;; 64 s) 65 fg_store=true 66 fg_all=false 67 fg_login=false 68 fg_bilan=false 69 ;; 70 v) 71 fg_verbose=true 72 ;; 73 :) 74 echo "$0: -"${OPTARG}" option: missing value" 75 exit 1 76 ;; 77 \?) 78 echo "$0: -"${OPTARG}" option: not supported" 79 exit 1 80 ;; 81 esac 82 done 83 shift $(($OPTIND-1)) 84 85 86 # Files and directories 87 # ===================== 88 LOCAL_DIR="/ccc/cont003/home/dsm/p86ipsl/CCC_MYPROJECT/output" 89 SAVE_DIR="/ccc/work/cont003/dsm/p86ipsl/CCC_MYPROJECT/" 90 91 if ( ${fg_dry} ) ; then 92 OUT_LOGIN="/dev/stdout" 93 OUT_BILAN="/dev/stdout" 94 OUT_STORE="/dev/stdout" 95 else 96 OUT_LOGIN="OUT_CONSO_LOGIN" 97 OUT_BILAN="OUT_CONSO_BILAN" 98 OUT_STORE="OUT_CONSO_STORE" 99 fi 100 101 #Today=$( date +%F ) 102 Today=$( ccc_myproject | grep gencmip6 | gawk '{print $NF}' ) 103 104 105 # Produce files 106 # ============= 107 108 cd ${LOCAL_DIR} 109 110 # 1- Conso par login (HOME) 111 # ------------------------- 112 # on garde la trace de chaque login, date en tete, en remplacant le 113 # fichier a chaque fois : OUT_CONSO_LOGIN 114 if ( ${fg_all} || ${fg_login} ) ; then 115 get_gencmip6_login | \ 116 gawk -v Today=$Today '{printf "%10s %-10s %10.2f hours\n", Today, $1, $2}' \ 117 > ${OUT_LOGIN} 118 fi 119 120 # 2- Conso total par jour 121 # ----------------------- 122 # on garde le total, date en tete en accumulant dans le fichier : 123 # OUT_CONSO_BILAN 124 if ( ${fg_all} || ${fg_bilan} ) ; then 125 printf "%10s " ${Today} >> ${OUT_BILAN} 126 get_gencmip6 Total | \ 127 gawk '{printf " %s %10.2f hours ", "total", $2}' \ 128 >> ${OUT_BILAN} 129 get_gencmip6 Allocated | \ 130 gawk '{printf " %s %10.2f hours ", "alloc", $2}' \ 131 >> ${OUT_BILAN} 132 get_gencmip6 Suggested | \ 133 gawk '{printf " %s %7s ", "use_theo", $NF}' \ 134 >> ${OUT_BILAN} 135 get_gencmip6 Real | \ 136 gawk '{printf " %s %7s ", "use_real", $NF}' \ 137 >> ${OUT_BILAN} 138 printf "\n" >> ${OUT_BILAN} 139 fi 140 141 # 3- volume cree sur STORE 142 # ------------------------ 143 # par login qui a consomme, en remplacant le fichier a chaque fois : 144 # OUT_CONSO_STORE 145 if ( ${fg_all} || ${fg_store} ) ; then 146 for login in $( get_gencmip6_login | \ 147 gawk '{if ($NF != 0.00) print $1}' ) ; do 148 printf "%10s %-10s %6s %s\n" \ 149 ${Today} ${login} \ 150 $( du -sbh $(ccc_home -A -u ${login})/IGCM_OUT/IPSLCM6 ) 151 done > ${OUT_STORE} 152 fi 153 154 155 # Save files (WORK) 156 # ================= 157 if ( ! ${fg_dry} ) ; then 158 Suffix=$( echo ${Today} | sed 's/-//g' ) 159 cp ${OUT_LOGIN} ${SAVE_DIR}/${OUT_LOGIN}_${Suffix} 160 cp ${OUT_BILAN} ${SAVE_DIR}/${OUT_BILAN}_${Suffix} 161 cp ${OUT_STORE} ${SAVE_DIR}/${OUT_STORE}_${Suffix} 162 fi 163 1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 # this must come first 5 from __future__ import print_function, unicode_literals, division 6 7 # standard library imports 8 from argparse import ArgumentParser 9 import os.path 10 import shutil 11 import json 12 13 14 ######################################## 15 def parse_myproject(filename, project): 16 17 logins = {} 18 19 with open(filename, "r") as filein: 20 # Skip lines until we find project name. 21 # Then extract script date 22 for ligne in filein: 23 if project["project"] in ligne: 24 today = ligne.split()[-1] 25 break 26 27 # Skip next two lines : first is blank and second is login titles 28 for _ in xrange(1): 29 next(filein) 30 31 # Login list 32 for ligne in filein: 33 if not ligne.strip(): 34 break 35 login, conso = ligne.split() 36 logins[login] = float(conso) 37 38 # Skip all the rest until we find total 39 for ligne in filein: 40 if "Total" in ligne: 41 total = float(ligne.split()[-1]) 42 break 43 44 # Skip all the rest until we find deadline 45 for ligne in filein: 46 if "Allocated" in ligne: 47 project["alloc"] = float(ligne.split()[-1]) 48 break 49 50 # Skip all the rest until we find deadline 51 for ligne in filein: 52 if "Suggested use at this time" in ligne: 53 utheo = float(ligne.split()[-1].strip("%")) 54 break 55 56 # Skip all the rest until we find deadline 57 for ligne in filein: 58 if "Real use at this time" in ligne: 59 ureal = float(ligne.split()[-1].strip("%")) 60 break 61 62 # Skip all the rest until we find deadline 63 for ligne in filein: 64 if "Project deadline" in ligne: 65 project["deadline"] = ligne.split()[-1] 66 break 67 68 return today, total, utheo, ureal, logins 69 70 71 ######################################## 72 def write_param(filename, project): 73 74 if args.dryrun: 75 print(json.dumps(project, indent=2)) 76 else: 77 with open(filename, "w") as fileout: 78 json.dump(project, fileout, indent=2) 79 80 81 ######################################## 82 def write_bilan(filename, today, total, ureal, utheo): 83 """ 84 Conso totale par jour 85 --------------------- 86 on garde le total, date en tete en accumulant dans le fichier : 87 OUT_CONSO_BILAN 88 """ 89 90 title_str = "{:10s} {:12s} {:11s} {:11s}\n".format( 91 "date", 92 "conso(hours)", 93 "real_use(%)", 94 "theo_use(%)", 95 ) 96 result_str = "{:10s} {:12.2f} {:11.2f} {:11.2f}\n".format( 97 today, 98 total, 99 ureal, 100 utheo, 101 ) 102 103 if args.dryrun: 104 print(title_str.strip()) 105 print(result_str.strip()) 106 else: 107 if not os.path.isfile(filename): 108 with open(filename, "w") as fileout: 109 fileout.write(title_str) 110 with open(filename, "a") as fileout: 111 fileout.write(result_str) 112 113 114 ######################################## 115 def write_utheo(filename, today, utheo): 116 """ 117 Conso théorique par jour 118 ------------------------ 119 OUT_CONSO_THEO 120 """ 121 122 title_str = "{:10s} {:11s}\n".format( 123 "date", 124 "theo_use(%)", 125 ) 126 result_str = "{:10s} {:11.2f}\n".format( 127 today, 128 utheo, 129 ) 130 131 if args.dryrun: 132 print(title_str.strip()) 133 print(result_str.strip()) 134 else: 135 if not os.path.isfile(filename): 136 with open(filename, "w") as fileout: 137 fileout.write(title_str) 138 with open(filename, "a") as fileout: 139 fileout.write(result_str) 140 141 142 ######################################## 143 def write_login(filename, today, logins): 144 """ 145 Conso par login (HOME) 146 ---------------------- 147 on garde la trace de chaque login, date en tete, en remplacant 148 le fichier a chaque fois : OUT_CONSO_LOGIN 149 """ 150 151 title_str = "{:10s} {:10s} {:12s}\n".format( 152 "date", 153 "login", 154 "conso(hours)", 155 ) 156 157 with open(filename, "w") as fileout: 158 if args.dryrun: 159 print(title_str.strip()) 160 else: 161 fileout.write(title_str) 162 163 for key in sorted(logins): 164 result_str = "{:10s} {:10s} {:12.2f}\n".format( 165 today, 166 key, 167 logins[key], 168 ) 169 if args.dryrun: 170 print(result_str.strip()) 171 else: 172 fileout.write(result_str) 173 174 175 ######################################## 176 def write_store(filename): 177 """ 178 volume cree sur STORE 179 --------------------- 180 par login qui a consomme, en remplacant le fichier a chaque fois : 181 OUT_CONSO_STORE 182 """ 183 184 pass 185 186 187 ######################################## 188 if __name__ == '__main__': 189 190 # Get arguments from command line 191 # =============================== 192 parser = ArgumentParser() 193 parser.add_argument("-v", "--verbose", action="store_true", 194 help="Verbose mode") 195 parser.add_argument("-d", "--dryrun", action="store_true", 196 help="dry run, no file produced") 197 parser.add_argument("-a", "--all", action="store_false", 198 help="produce all files (default)") 199 parser.add_argument("-b", "--bilan", action="store_true", 200 help="produce all files (default)") 201 parser.add_argument("-l", "--login", action="store_true", 202 help="produce all files (default)") 203 parser.add_argument("-s", "--store", action="store_true", 204 help="produce all files (default)") 205 206 args = parser.parse_args() 207 if args.verbose: 208 print(os.path.basename(__file__)) 209 print(args) 210 211 # Files and directories 212 # ===================== 213 LOCAL_DIR = os.path.join( 214 "/ccc", "cont003", "home", "dsm", "p86ipsl", "ConsoGENCMIP6", "output" 215 # "/home_local", "slipsl", "ConsoGENCMIP6", "output" 216 ) 217 SAVE_DIR = os.path.join( 218 "/ccc", "work", "cont003", "dsm", "p86ipsl", "ConsoGENCMIP6", 219 # "/home_local", "slipsl", "ConsoGENCMIP6", "save" 220 ) 221 222 OUT_PARAM = "OUT_CONSO_PARAM" 223 OUT_BILAN = "OUT_CONSO_BILAN" 224 OUT_UTHEO = "OUT_CONSO_UTHEO" 225 OUT_LOGIN = "OUT_CONSO_LOGIN" 226 OUT_STORE = "OUT_CONSO_STORE" 227 228 if args.verbose: 229 print(LOCAL_DIR) 230 print(SAVE_DIR) 231 232 ccc_myproject = "ccc_myproject.dat" 233 234 project = {} 235 project["project"] = "gencmip6" 236 237 (today, total, utheo, ureal, logins) = \ 238 parse_myproject( 239 os.path.join(LOCAL_DIR, ccc_myproject), 240 project 241 ) 242 243 if args.verbose: 244 print(today, utheo, ureal) 245 print(project) 246 print(logins) 247 248 # Produce files 249 # ============= 250 251 # 1- Parametres du projet 252 # ----------------------- 253 write_param(os.path.join(LOCAL_DIR, OUT_PARAM), project) 254 255 # 2- Conso totale par jour 256 # ------------------------ 257 write_bilan( 258 os.path.join(LOCAL_DIR, OUT_BILAN), 259 today, 260 total, 261 ureal, 262 utheo 263 ) 264 265 # 2b- Conso théorique par jour 266 # ---------------------------- 267 write_utheo(os.path.join(LOCAL_DIR, OUT_UTHEO), today, utheo) 268 269 # 3- Conso par login (HOME) 270 # ------------------------- 271 write_login(os.path.join(LOCAL_DIR, OUT_LOGIN), today, logins) 272 273 # 4- volume cree sur STORE 274 # ------------------------ 275 write_store(os.path.join(LOCAL_DIR, OUT_STORE)) 276 277 # Save files (on WORKDIR) 278 # ======================= 279 if not args.dryrun: 280 suffix = today.replace("-", "") 281 file_list = [ 282 OUT_PARAM, 283 OUT_BILAN, 284 OUT_UTHEO, 285 OUT_LOGIN, 286 OUT_STORE 287 ] 288 for filename in file_list: 289 filein = os.path.join(LOCAL_DIR, filename) 290 if os.path.isfile(filein): 291 fileout = os.path.join(SAVE_DIR, "_".join((filename, suffix))) 292 shutil.copy(filein, fileout)
Note: See TracChangeset
for help on using the changeset viewer.