#!/usr/bin/env python # -*- coding: utf-8 -*- # ==================================================================== # # Author: Sonia Labetoulle # # Contact: sonia.labetoulle _at_ ipsl.jussieu.fr # # Created: 2016 # # History: # # Modification: # # ==================================================================== # # this must come first # from __future__ import print_function, unicode_literals, division from __future__ import print_function, division # standard library imports import socket import os import os.path import glob import subprocess import datetime as dt import numpy as np import ConfigParser as cp # Application library imports ######################################################################## # .. Constants and variables .. # ============================= cm2in = 1. / 2.54 ######################################################################## def dods_cp(img_out, img_name, DODS): """ """ if not DODS["DIR"]: print("DODS directory not defined") return # .. Convert pdf to temporary png .. # ================================== import matplotlib.pyplot as plt img_png = img_out.replace(".pdf", "_tmp.png") dpi = 200. plt.savefig( img_png, dpi=dpi, format="png", # transparent=True, # bbox_inches="tight", # pad_inches=0.2 * cm2in, ) # .. Copy files to dods server .. # =============================== # ... pdf file ... # ---------------- command = [ "scp", img_out, "{}@{}:{}".format( DODS["USER"], DODS["SERVER"], os.path.join(DODS["DIR"], "pdf", img_name + ".pdf") ) ] print(command) try: subprocess.call(command) except Exception as rc: print("Error in scp for {}:\n{}".format(command, rc)) # ... png file ... # ---------------- command = [ "scp", img_png, "{}@{}:{}".format( DODS["USER"], DODS["SERVER"], os.path.join(DODS["DIR"], "img", img_name + ".png") ) ] print(command) try: subprocess.call(command) except Exception as rc: print("Error in scp for {}:\n{}".format(command, rc)) # .. Delete temporary png file .. # =============================== try: os.remove(img_png) except Exception as rc: print("Could not remove {}:\n{}".format(img_png, rc)) return ######################################################################## def parse_config(filename): DIR = {} DODS = {} config = cp.ConfigParser(allow_no_value=True) config.optionxform = str config.read(filename) for section in ("projet", "directories"): if not config.has_section(section): print( "Missing section {} in {}, we stop".format(section, filename) ) exit(9) # ... Project name ... # -------------------- section = "projet" option = "name" # project_name = config.get(section, option) # ... Common directories ... # -------------------------- section = "directories" for option in config.options(section): DIR[option] = config.get(section, option) if DIR[option] and not os.path.isdir(DIR[option]): print("mkdir {}".format(DIR[option])) try: os.makedirs(DIR[option]) except Exception as rc: print("Could not create {}:\n{}".format(DIR[option], rc)) # .. DODS configuration ... # ------------------------- section = "dods" for option in config.options(section): DODS[option] = config.get(section, option) return (DIR, DODS) ######################################################################## def string_to_percent(x): """ """ return float(x.strip("%"))/100. ######################################################################## def string_to_size_unit(x): """ """ if unicode(x).isdecimal(): x = x + "o" (size, unit) = (float(x[:-1]), x[-1]) return SizeUnit(size, unit) ######################################################################## def string_to_float(x): """ """ return float(x.strip("h")) ######################################################################## def string_to_date(ssaammjj, fmt="%Y%m%d"): """ """ fmts = ["%Y%m%d", "%Y-%m-%d", "%Y_%m_%d"] for fmt in fmts: try: res = dt.datetime.strptime(ssaammjj, fmt) except Exception: pass else: break return res.date() ######################################################################## def string_to_datetime(string, fmt="%Y-%m-%d-%H:%M"): """ """ return dt.datetime.strptime(string, fmt) ######################################################################## def where_we_run(): res = "" if "curie" in socket.getfqdn(): res = "curie" elif "ipsl" in socket.getfqdn(): res = "ipsl" else: res = "default" return res ######################################################################## def get_last_file(dir_data, pattern): """ """ current_dir = os.getcwd() os.chdir(dir_data) filename = pattern + "*" file_list = glob.glob(os.path.join(dir_data, filename)) if file_list: res = sorted(file_list)[-1] else: res = None os.chdir(current_dir) return res ######################################################################## def get_input_files(dir_data, file_list): """ """ res = [] for filename in file_list: res.append(get_last_file(dir_data, filename)) if None in res: print("\nMissing one or more input files, we stop.") for f_in, f_out in zip(file_list, res): print("=> {}: {}".format(f_in, f_out)) exit(9) return res ######################################################################## def plot_save(img_out, title, png=False): """ """ import matplotlib.pyplot as plt from matplotlib.backends.backend_pdf import PdfPages dpi = 200. dirname = os.path.dirname(img_out) if not os.path.isdir(dirname): print("mkdir {}".format(dirname)) try: os.makedirs(dirname) except Exception as rc: print("Could not create {}:\n{}".format(dirname, rc)) if png: img_png = img_out.replace(".pdf", ".png") plt.savefig( img_png, dpi=dpi, format="png", # transparent=True, # bbox_inches="tight", # pad_inches=0.2 * cm2in, ) with PdfPages(img_out) as pdf: pdf.savefig(dpi=dpi) # ... pdf file's metadata ... # --------------------------- d = pdf.infodict() d["Title"] = title d["Author"] = os.path.basename(__file__) # d["Subject"] = ( # "Time spent over specific commands during create_ts " # "jobs at IDRIS and four configurations at TGCC" # ) # d["Keywords"] = "bench create_ts TGCC IDRIS ncrcat" # d["CreationDate"] = dt.datetime(2009, 11, 13) # d["ModDate"] = dt.datetime.today() ######################################################################## class AllocItem(object): # -------------------------------------------------------------------- def __init__( self, alloc_id, machine, node_type, start_date, end_date, alloc ): self.alloc_id = alloc_id self.machine = machine self.node_type = node_type self.start_date = start_date self.end_date = end_date self.alloc = alloc delta = self.end_date - self.start_date self.days = delta.days + 1 self.daily_conso = self.alloc / self.days # -------------------------------------------------------------------- def __repr__(self): return "{} ({:%Y%m%d}/{:%Y%m%d}): {}".format( self.machine, self.start_date, self.end_date, self.alloc, ) ######################################################################## class Project(object): # -------------------------------------------------------------------- def __init__(self, project_name, center): self.project = project_name self.centre = center self.alloc_items = [] # -------------------------------------------------------------------- def __repr__(self): return "{}/{}: {}".format( self.project, self.centre, self.alloc_items, ) # -------------------------------------------------------------------- def add_alloc( self, alloc_id, machine, node_type, start_date, end_date, alloc ): alloc_item = AllocItem( alloc_id, machine, node_type, start_date, end_date, alloc ) self.alloc_items.append(alloc_item) self.start_date = min( [item.start_date for item in self.alloc_items] ) self.end_date = max( [item.end_date for item in self.alloc_items] ) self.alloc = sum( [item.alloc for item in self.alloc_items] ) self.max_daily_conso = max( [item.daily_conso for item in self.alloc_items] ) self.days = sum( [item.days for item in self.alloc_items] ) self.nb_alloc = len(self.alloc_items) # -------------------------------------------------------------------- def get_theo_eq(self, dates): # x0 = 0 y0 = 0. for item in self.alloc_items: yi = y0 yf = y0 + item.alloc / self.alloc if item.start_date.date() in dates: xi = dates.index(item.start_date.date()) else: xi = 0 if item.end_date.date() in dates: xf = dates.index(item.end_date.date()) else: xf = len(dates) + 1 m = np.array([[xi, 1.], [xf+1, 1.]]) n = np.array([yi, yf]) try: polynome = np.poly1d(np.linalg.solve(m, n)) except np.linalg.linalg.LinAlgError: print("error poly") item.theo_eq = None else: item.theo_eq = polynome item.xi = xi item.xf = xf item.yi = yi item.yf = yf y0 = yf # -------------------------------------------------------------------- def fill_data(self, row): # self.id = row["id"] # self.machine = row["machine"] # self.node = row["node_type"] # self.start_date = row["start_date"] # self.end_date = row["end_date"] # self.alloc = row["total_hrs"] delta = self.end_date - self.start_date self.days = delta.days + 1 ######################################################################## class SizeUnit(object): # -------------------------------------------------------------------- def __init__(self, size, unit): self.size = size self.unit = unit # -------------------------------------------------------------------- def __repr__(self): return "{:6.2f}{}o".format(self.size, self.unit) # -------------------------------------------------------------------- def convert_size(self, unit_out): """ """ prefixes = ["o", "K", "M", "G", "T", "P", "H"] if not self.size or \ self.unit == unit_out: size_out = self.size else: idx_deb = prefixes.index(self.unit) idx_fin = prefixes.index(unit_out) size_out = self.size for i in xrange(abs(idx_fin-idx_deb)): if idx_fin > idx_deb: size_out = size_out / 1024 else: size_out = size_out * 1024 return SizeUnit(size_out, unit_out) ######################################################################## if __name__ == '__main__': pass