Changeset 2413
- Timestamp:
- 02/10/15 12:25:07 (10 years ago)
- Location:
- TOOLS/ConsoGENCMIP6/bin
- Files:
-
- 3 added
- 1 edited
- 8 moved
Legend:
- Unmodified
- Added
- Removed
-
TOOLS/ConsoGENCMIP6/bin/conso_gencmip6.py
r2412 r2413 7 7 # standard library imports 8 8 from argparse import ArgumentParser 9 import json 10 import shutil 11 import os 9 12 import os.path 10 import shutil 11 import json 12 13 14 ######################################## 15 def parse_myproject(filename, project): 16 13 import subprocess 14 # import datetime as dt 15 16 # Application library imports 17 from gencmip6 import * 18 from gencmip6_path import * 19 20 21 ######################################## 22 def get_storedir(login): 23 24 command = ["ccc_home", "-A", "-u", login] 25 try : 26 res = subprocess.check_output(command) 27 except Exception as rc : 28 # print(rc) 29 res = None 30 31 return res 32 33 34 ######################################## 35 def get_dirsize(dirname): 36 37 command = ["du", "-sbh", dirname] 38 try : 39 res = float(subprocess.check_output(command)) 40 except Exception as rc : 41 print(rc) 42 res = None 43 44 return res 45 46 47 # ######################################## 48 # def get_dirlist(dirname): 49 50 51 # return output 52 53 54 ######################################## 55 def parse_myproject(filename): 56 57 project = {} 58 project["project"] = "gencmip6" 17 59 logins = {} 60 61 if where_we_run() == "curie": 62 try : 63 res = subprocess.check_output("ccc_myproject") 64 except Exception as rc : 65 print(rc) 66 exit() 67 with open(os.path.join(DIR["DATA"], OUT_CCCMP), "w") as fileout: 68 fileout.write(res) 18 69 19 70 with open(filename, "r") as filein: … … 23 74 if project["project"] in ligne: 24 75 today = ligne.split()[-1] 76 today = string_to_date(today) 77 25 78 break 26 79 … … 29 82 next(filein) 30 83 31 # Login list 84 # Login list, until blank line 32 85 for ligne in filein: 33 86 if not ligne.strip(): … … 36 89 logins[login] = float(conso) 37 90 38 # Skip all the rest until we find total91 # Skip until we find consumed time (hours) 39 92 for ligne in filein: 40 93 if "Total" in ligne: … … 42 95 break 43 96 44 # Skip all the rest until we find deadline97 # Skip until we find allocated time (hours) 45 98 for ligne in filein: 46 99 if "Allocated" in ligne: … … 48 101 break 49 102 50 # Skip all the rest until we find deadline103 # Skip until we find theoratical use (%) 51 104 for ligne in filein: 52 105 if "Suggested use at this time" in ligne: … … 54 107 break 55 108 56 # Skip all the rest until we find deadline109 # Skip until we find real use (%) 57 110 for ligne in filein: 58 111 if "Real use at this time" in ligne: … … 60 113 break 61 114 62 # Skip all the restuntil we find deadline115 # Skip until we find deadline 63 116 for ligne in filein: 64 117 if "Project deadline" in ligne: … … 66 119 break 67 120 68 return today, total, utheo, ureal, logins121 return project, logins, today, total, utheo, ureal 69 122 70 123 … … 94 147 "theo_use(%)", 95 148 ) 96 result_str = "{: 10s} {:12.2f} {:11.2f} {:11.2f}\n".format(149 result_str = "{:%Y-%m-%d} {:12.2f} {:11.2f} {:11.2f}\n".format( 97 150 today, 98 151 total, … … 124 177 "theo_use(%)", 125 178 ) 126 result_str = "{: 10s} {:11.2f}\n".format(179 result_str = "{:%Y-%m-%d} {:11.2f}\n".format( 127 180 today, 128 181 utheo, … … 162 215 163 216 for key in sorted(logins): 164 result_str = "{: 10s} {:10s} {:12.2f}\n".format(217 result_str = "{:%Y-%m-%d} {:10s} {:12.2f}\n".format( 165 218 today, 166 219 key, … … 174 227 175 228 ######################################## 176 def write_store(filename ):229 def write_store(filename, logins): 177 230 """ 178 231 volume cree sur STORE … … 182 235 """ 183 236 184 pass 237 items = (login for login, conso in logins.iteritems() 238 if conso > 0.) 239 240 for login in items: 241 storedir = get_storedir(login) 242 if not storedir: 243 break 244 igcm_out = os.path.join(storedir, "IGCM_OUT") 245 246 for dirname in os.listdir(igcm_out): 247 print(dirname, get_dirsize(dirname)) 185 248 186 249 … … 207 270 if args.verbose: 208 271 print(os.path.basename(__file__)) 272 print(where_we_run()) 209 273 print(args) 210 274 211 # Files and directories 212 # ===================== 213 LOCAL_DIR = os.path.join( 214 "/ccc", "cont003", "home", "dsm", "p86ipsl", "ConsoGENCMIP6", "output" 215 # "/home_local", "slipsl", "ConsoGENCMIP6", "output" 216 ) 217 SAVE_DIR = os.path.join( 218 "/ccc", "work", "cont003", "dsm", "p86ipsl", "ConsoGENCMIP6", 219 # "/home_local", "slipsl", "ConsoGENCMIP6", "save" 220 ) 275 # # Files and directories 276 # # ===================== 277 # if where_we_run() == "curie": 278 # LOCAL_DIR = os.path.join( 279 # "/ccc", 280 # "cont003", 281 # "home", 282 # "dsm", 283 # "p86ipsl", 284 # "ConsoGENCMIP6", 285 # "output" 286 # ) 287 # SAVE_DIR = os.path.join( 288 # "/ccc", 289 # "work", 290 # "cont003", 291 # "dsm", 292 # "p86ipsl", 293 # "ConsoGENCMIP6", 294 # ) 295 # # elif where_we_run() == "ipsl": 296 # # LOCAL_DIR = os.path.join( 297 # # "/home_local", 298 # # "slipsl", 299 # # "ConsoGENCMIP6", 300 # # "output" 301 # # ) 302 # # SAVE_DIR = os.path.join( 303 # # "/home_local", 304 # # "slipsl", 305 # # "ConsoGENCMIP6", 306 # # "save" 307 # # ) 308 # else: 309 # LOCAL_DIR = os.path.join( 310 # "..", 311 # "output" 312 # ) 313 # SAVE_DIR = os.path.join( 314 # "..", 315 # "save" 316 # ) 317 318 # LOCAL_DIR = os.path.abspath(LOCAL_DIR) 319 # SAVE_DIR = os.path.abspath(SAVE_DIR) 221 320 222 321 OUT_PARAM = "OUT_CONSO_PARAM" … … 225 324 OUT_LOGIN = "OUT_CONSO_LOGIN" 226 325 OUT_STORE = "OUT_CONSO_STORE" 326 OUT_CCCMP = "ccc_myproject.dat" 227 327 228 328 if args.verbose: 229 print(LOCAL_DIR) 230 print(SAVE_DIR) 231 232 ccc_myproject = "ccc_myproject.dat" 233 234 project = {} 235 project["project"] = "gencmip6" 236 237 (today, total, utheo, ureal, logins) = \ 238 parse_myproject( 239 os.path.join(LOCAL_DIR, ccc_myproject), 240 project 241 ) 329 # print(LOCAL_DIR) 330 # print(SAVE_DIR) 331 print(DIR["DATA"]) 332 print(DIR["SAVE"]) 333 334 (project, logins, today, total, utheo, ureal) = \ 335 parse_myproject(os.path.join(DIR["DATA"], OUT_CCCMP)) 242 336 243 337 if args.verbose: … … 251 345 # 1- Parametres du projet 252 346 # ----------------------- 253 write_param(os.path.join( LOCAL_DIR, OUT_PARAM), project)347 write_param(os.path.join(DIR["DATA"], OUT_PARAM), project) 254 348 255 349 # 2- Conso totale par jour 256 350 # ------------------------ 257 351 write_bilan( 258 os.path.join( LOCAL_DIR, OUT_BILAN),352 os.path.join(DIR["DATA"], OUT_BILAN), 259 353 today, 260 354 total, … … 265 359 # 2b- Conso théorique par jour 266 360 # ---------------------------- 267 write_utheo(os.path.join( LOCAL_DIR, OUT_UTHEO), today, utheo)361 write_utheo(os.path.join(DIR["DATA"], OUT_UTHEO), today, utheo) 268 362 269 363 # 3- Conso par login (HOME) 270 364 # ------------------------- 271 write_login(os.path.join( LOCAL_DIR, OUT_LOGIN), today, logins)365 write_login(os.path.join(DIR["DATA"], OUT_LOGIN), today, logins) 272 366 273 367 # 4- volume cree sur STORE 274 368 # ------------------------ 275 write_store(os.path.join(LOCAL_DIR, OUT_STORE)) 369 # if where_we_run() == "curie": 370 # write_store(os.path.join(DIR["DATA"], OUT_STORE)) 371 write_store(os.path.join(DIR["DATA"], OUT_STORE), logins) 276 372 277 373 # Save files (on WORKDIR) 278 374 # ======================= 279 375 if not args.dryrun: 280 suffix = today.replace("-", "")376 suffix = "{:%Y%m%d}".format(today) 281 377 file_list = [ 282 378 OUT_PARAM, … … 287 383 ] 288 384 for filename in file_list: 289 filein = os.path.join( LOCAL_DIR, filename)385 filein = os.path.join(DIR["DATA"], filename) 290 386 if os.path.isfile(filein): 291 fileout = os.path.join( SAVE_DIR, "_".join((filename, suffix)))387 fileout = os.path.join(DIR["SAVE"], "_".join((filename, suffix))) 292 388 shutil.copy(filein, fileout) -
TOOLS/ConsoGENCMIP6/bin/gencmip6.py
r2411 r2413 7 7 # standard library imports 8 8 # from argparse import ArgumentParser 9 import socket 9 10 import os 10 11 import os.path … … 53 54 54 55 ######################################## 56 def where_we_run(): 57 58 res = "" 59 if "curie" in socket.getfqdn(): 60 res = "curie" 61 elif "ipsl" in socket.getfqdn(): 62 res = "ipsl" 63 else: 64 res = "default" 65 66 return res 67 68 69 ######################################## 55 70 def get_last_file(dir_data, pattern): 56 71 """ … … 59 74 os.chdir(dir_data) 60 75 filename = pattern + "*" 61 return_value = sorted(glob.glob(os.path.join(dir_data, filename)))[-1] 76 file_list = glob.glob(os.path.join(dir_data, filename)) 77 if file_list: 78 res = sorted(file_list)[-1] 79 else: 80 res = None 62 81 os.chdir(current_dir) 63 return re turn_value82 return res 64 83 65 84 … … 99 118 n = np.array([utheos[x1], utheos[x2]]) 100 119 101 (a, b) = np.linalg.solve(m, n) 120 try: 121 (a, b) = np.linalg.solve(m, n) 122 except np.linalg.linalg.LinAlgError: 123 (a, b) = (None, None) 102 124 103 delta = int(round((-b/a)-x1 + 1)) 125 if a and b: 126 delta = int(round((-b/a)-x1 + 1)) 104 127 105 d1 = dates[x1] 106 self.date_init = d1 + dt.timedelta(days=delta) 128 d1 = dates[x1] 129 self.date_init = d1 + dt.timedelta(days=delta) 130 else: 131 self.date_init = dt.datetime(self.deadline.year, 1, 1) 107 132 108 133 -
TOOLS/ConsoGENCMIP6/bin/plot_bilan.py
r2411 r2413 9 9 import os 10 10 import os.path 11 import glob11 # import glob 12 12 import datetime as dt 13 13 import numpy as np … … 15 15 from matplotlib.backends.backend_pdf import PdfPages 16 16 17 # Application library imports 17 18 from gencmip6 import * 19 from gencmip6_path import * 18 20 19 21 … … 168 170 if args.full: 169 171 line_style = "-" 172 line_width = 0.1 170 173 else: 171 174 line_style = "+-" 175 line_width = 0.2 172 176 173 177 ax_conso.bar(xcoord, consos, align="center", color="linen", 174 linewidth= 0.2, label="conso (heures)")178 linewidth=line_width, label="conso (heures)") 175 179 176 180 ax_theo.plot(xcoord, theo_equs, "--", … … 269 273 # ... Files and directories ... 270 274 # ----------------------------- 271 dir_data = os.path.join("..", "output")272 275 file_pattern = "OUT_CONSO_" 273 file_param = get_last_file(dir_data, file_pattern+"PARAM") 274 file_utheo = get_last_file(dir_data, file_pattern+"UTHEO") 275 file_bilan = get_last_file(dir_data, file_pattern+"BILAN") 276 file_login = get_last_file(dir_data, file_pattern+"LOGIN") 277 file_store = get_last_file(dir_data, file_pattern+"STORE") 276 file_param = get_last_file(DIR["DATA"], file_pattern+"PARAM") 277 file_utheo = get_last_file(DIR["DATA"], file_pattern+"UTHEO") 278 file_bilan = get_last_file(DIR["DATA"], file_pattern+"BILAN") 279 file_login = get_last_file(DIR["DATA"], file_pattern+"LOGIN") 280 file_store = get_last_file(DIR["DATA"], file_pattern+"STORE") 281 img_name = "bilan.pdf" 278 282 279 283 if args.verbose: … … 363 367 # ... Save figure ... 364 368 # ------------------- 365 dirout = "img" 366 img_name = "bilan.pdf" 367 plot_save(os.path.join(dirout, img_name)) 369 plot_save(os.path.join(DIR["PLOT"], img_name)) 368 370 369 371 plt.show() -
TOOLS/ConsoGENCMIP6/bin/plot_login.py
r2411 r2413 9 9 import os 10 10 import os.path 11 import glob12 import datetime as dt11 # import glob 12 # import datetime as dt 13 13 import numpy as np 14 14 import matplotlib.pyplot as plt 15 15 from matplotlib.backends.backend_pdf import PdfPages 16 16 17 # Application library imports 17 18 from gencmip6 import * 18 19 20 # ######################################## 21 # def string_to_percent(x): 22 # """ 23 # """ 24 # return float(x.strip("%"))/100. 25 26 27 # ######################################## 28 # def string_to_float(x): 29 # """ 30 # """ 31 # return float(x.strip("h")) 32 33 34 # ######################################## 35 # def string_to_date(ssaammjj, fmt="%Y-%m-%d"): 36 # """ 37 # """ 38 # return dt.datetime.strptime(ssaammjj, fmt) 39 40 41 # ######################################## 42 # def date_to_string(dtdate, fmt="%Y-%m-%d"): 43 # """ 44 # """ 45 # return dt.datetime.strftime(dtdate, fmt) 46 47 48 # ######################################## 49 # def get_last_file(dir_data, pattern): 50 # """ 51 # """ 52 # current_dir = os.getcwd() 53 # os.chdir(dir_data) 54 # filename = pattern + "*" 55 # return_value = sorted(glob.glob(os.path.join(dir_data, filename)))[-1] 56 # os.chdir(current_dir) 57 # return return_value 58 59 60 # ######################################## 61 # class Project(object): 62 63 # #--------------------------------------- 64 # def __init__(self): 65 # self.project = "" 66 # self.date_init = "" 67 # self.deadline = "" 68 # self.alloc = 0 69 70 # #--------------------------------------- 71 # def fill_data(self, filein): 72 # import json 73 # dico = json.load(open(filein, "r")) 74 # self.project = dico["project"] 75 # self.deadline = string_to_date(dico["deadline"]) + \ 76 # dt.timedelta(days=-1) 77 # self.alloc = dico["alloc"] 78 79 # #--------------------------------------- 80 # def get_date_init(self, filein): 81 # data = np.genfromtxt( 82 # filein, 83 # skip_header=1, 84 # converters={0: string_to_date, 85 # 1: string_to_percent}, 86 # missing_values="nan", 87 # ) 88 # dates, utheos = zip(*data) 89 90 # (x1, x2) = (np.nanargmin(utheos), np.nanargmax(utheos)) 91 92 # m = np.array([[x1, 1.], [x2, 1.]]) 93 # n = np.array([utheos[x1], utheos[x2]]) 94 95 # (a, b) = np.linalg.solve(m, n) 96 97 # delta = int(round((-b/a)-x1 + 1)) 98 99 # d1 = dates[x1] 100 # self.date_init = d1 + dt.timedelta(days=delta) 19 from gencmip6_path import * 101 20 102 21 … … 262 181 # ... Files and directories ... 263 182 # ----------------------------- 264 dir_data = os.path.join("..", "output")265 183 file_pattern = "OUT_CONSO_" 266 file_param = get_last_file(dir_data, file_pattern+"PARAM") 267 file_utheo = get_last_file(dir_data, file_pattern+"UTHEO") 268 file_bilan = get_last_file(dir_data, file_pattern+"BILAN") 269 file_login = get_last_file(dir_data, file_pattern+"LOGIN") 270 file_store = get_last_file(dir_data, file_pattern+"STORE") 184 file_param = get_last_file(DIR["DATA"], file_pattern+"PARAM") 185 file_utheo = get_last_file(DIR["DATA"], file_pattern+"UTHEO") 186 file_bilan = get_last_file(DIR["DATA"], file_pattern+"BILAN") 187 file_login = get_last_file(DIR["DATA"], file_pattern+"LOGIN") 188 file_store = get_last_file(DIR["DATA"], file_pattern+"STORE") 189 img_name = "login.pdf" 271 190 272 191 # .. Get project info .. … … 324 243 # ... Save figure ... 325 244 # ------------------- 326 dirout = "img" 327 img_name = "login.pdf" 328 plot_save(os.path.join(dirout, img_name)) 245 plot_save(os.path.join(DIR["PLOT"], img_name)) 329 246 330 247 plt.show() -
TOOLS/ConsoGENCMIP6/bin/plot_store.py
r2411 r2413 15 15 from matplotlib.backends.backend_pdf import PdfPages 16 16 17 # Application library imports 17 18 from gencmip6 import * 18 19 20 # ######################################## 21 # def string_to_percent(x): 22 # """ 23 # """ 24 # return float(x.strip("%"))/100. 25 26 27 # ######################################## 28 # def string_to_size_unit(x): 29 # """ 30 # """ 31 # (size, unit) = (float(x[:-1]), x[-1]) 32 # return SizeUnit(size, unit) 33 34 35 # ######################################## 36 # def string_to_float(x): 37 # """ 38 # """ 39 # return float(x.strip("h")) 40 41 42 # ######################################## 43 # def string_to_date(ssaammjj, fmt="%Y-%m-%d"): 44 # """ 45 # """ 46 # return dt.datetime.strptime(ssaammjj, fmt) 47 48 49 # ######################################## 50 # def date_to_string(dtdate, fmt="%Y-%m-%d"): 51 # """ 52 # """ 53 # return dt.datetime.strftime(dtdate, fmt) 54 55 56 # ######################################## 57 # def get_last_file(dir_data, pattern): 58 # """ 59 # """ 60 # current_dir = os.getcwd() 61 # os.chdir(dir_data) 62 # filename = pattern + "*" 63 # return_value = sorted(glob.glob(os.path.join(dir_data, filename)))[-1] 64 # os.chdir(current_dir) 65 # return return_value 66 67 68 # ######################################## 69 # class Project(object): 70 71 # #--------------------------------------- 72 # def __init__(self): 73 # self.project = "" 74 # self.date_init = "" 75 # self.deadline = "" 76 # self.alloc = 0 77 78 # #--------------------------------------- 79 # def fill_data(self, filein): 80 # import json 81 # dico = json.load(open(filein, "r")) 82 # self.project = dico["project"] 83 # self.deadline = string_to_date(dico["deadline"]) + \ 84 # dt.timedelta(days=-1) 85 # self.alloc = dico["alloc"] 86 87 # #--------------------------------------- 88 # def get_date_init(self, filein): 89 # data = np.genfromtxt( 90 # filein, 91 # skip_header=1, 92 # converters={0: string_to_date, 93 # 1: string_to_percent}, 94 # missing_values="nan", 95 # ) 96 # dates, utheos = zip(*data) 97 98 # (x1, x2) = (np.nanargmin(utheos), np.nanargmax(utheos)) 99 100 # m = np.array([[x1, 1.], [x2, 1.]]) 101 # n = np.array([utheos[x1], utheos[x2]]) 102 103 # (a, b) = np.linalg.solve(m, n) 104 105 # delta = int(round((-b/a)-x1 + 1)) 106 107 # d1 = dates[x1] 108 # self.date_init = d1 + dt.timedelta(days=delta) 109 110 111 # ######################################## 112 # class SizeUnit(object): 113 # #--------------------------------------- 114 # def __init__(self, size, unit): 115 # self.size = size 116 # self.unit = unit 117 118 # #--------------------------------------- 119 # def __repr__(self): 120 # return "{:6.2f}{}o".format(self.size, self.unit) 121 122 # #--------------------------------------- 123 # def convert_size(self, unit_out): 124 # """ 125 # """ 126 # prefixes = ["K", "M", "G", "T", "P", "H"] 127 128 # if not self.size or \ 129 # self.unit == unit_out: 130 # size_out = self.size 131 # else: 132 # idx_deb = prefixes.index(self.unit) 133 # idx_fin = prefixes.index(unit_out) 134 # size_out = self.size 135 # for i in xrange(abs(idx_fin-idx_deb)): 136 # if idx_fin > idx_deb: 137 # size_out = size_out / 1024 138 # else: 139 # size_out = size_out * 1024 140 141 # return SizeUnit(size_out, unit_out) 19 from gencmip6_path import * 142 20 143 21 … … 199 77 """ 200 78 """ 201 # items = (item for item in self.itervalues() if item.dir)202 79 items = (subitem for item in self.itervalues() 203 80 for subitem in item.listdir … … 336 213 # ... Files and directories ... 337 214 # ----------------------------- 338 dir_data = os.path.join("..", "output")339 215 file_pattern = "OUT_CONSO_" 340 file_param = get_last_file(dir_data, file_pattern+"PARAM") 341 file_utheo = get_last_file(dir_data, file_pattern+"UTHEO") 342 file_bilan = get_last_file(dir_data, file_pattern+"BILAN") 343 file_login = get_last_file(dir_data, file_pattern+"LOGIN") 344 file_store = get_last_file(dir_data, file_pattern+"STORE") 216 file_param = get_last_file(DIR["DATA"], file_pattern+"PARAM") 217 file_utheo = get_last_file(DIR["DATA"], file_pattern+"UTHEO") 218 file_bilan = get_last_file(DIR["DATA"], file_pattern+"BILAN") 219 file_login = get_last_file(DIR["DATA"], file_pattern+"LOGIN") 220 file_store = get_last_file(DIR["DATA"], file_pattern+"STORE") 221 img_name = "store.pdf" 345 222 346 223 # .. Get project info .. … … 407 284 # ... Save figure ... 408 285 # ------------------- 409 dirout = "img" 410 img_name = "store.pdf" 411 plot_save(os.path.join(dirout, img_name)) 286 plot_save(os.path.join(DIR["PLOT"], img_name)) 412 287 413 288 plt.show()
Note: See TracChangeset
for help on using the changeset viewer.