source: TOOLS/ConsoGENCI/trunk/bin/libconso.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# ==================================================================== #
# Author: Sonia Labetoulle                                             #
# Contact: sonia.labetoulle _at_ ipsl.jussieu.fr                       #
# Created: 2016                                                        #
# History:                                                             #
# Modification:                                                        #
# ==================================================================== #

# this must come first
# from __future__ import print_function, unicode_literals, division
from __future__ import print_function, division

# standard library imports
import socket
import os
import os.path
import glob
import subprocess
import datetime as dt
import numpy as np
import ConfigParser as cp

# Application library imports

########################################################################

# .. Constants and variables ..
# =============================
cm2in = 1. / 2.54


########################################################################
def dods_cp(img_out, img_name, DODS):
  """
  """
  if not DODS["DIR"]:
    print("DODS directory not defined")
    return

  # .. Convert pdf to temporary png ..
  # ==================================
  import matplotlib.pyplot as plt

  img_png = img_out.replace(".pdf", "_tmp.png")
  dpi = 200.

  plt.savefig(
    img_png,
    dpi=dpi,
    format="png",
    # transparent=True,
    # bbox_inches="tight",
    # pad_inches=0.2 * cm2in,
  )

  # .. Copy files to dods server ..
  # ===============================
  # ... pdf file ...
  # ----------------
  command = [
    "scp",
    img_out,
    "{}@{}:{}".format(
      DODS["USER"],
      DODS["SERVER"],
      os.path.join(DODS["DIR"], "pdf", img_name + ".pdf")
    )
  ]
  print(command)
  try:
    subprocess.call(command)
  except Exception as rc:
    print("Error in scp for {}:\n{}".format(command, rc))

  # ... png file ...
  # ----------------
  command = [
    "scp",
    img_png,
    "{}@{}:{}".format(
      DODS["USER"],
      DODS["SERVER"],
      os.path.join(DODS["DIR"], "img", img_name + ".png")
    )
  ]

  print(command)
  try:
    subprocess.call(command)
  except Exception as rc:
    print("Error in scp for {}:\n{}".format(command, rc))

  # .. Delete temporary png file ..
  # ===============================
  try:
    os.remove(img_png)
  except Exception as rc:
    print("Could not remove {}:\n{}".format(img_png, rc))

  return


########################################################################
def parse_config(filename):

  DIR = {}
  DODS = {}

  config = cp.ConfigParser(allow_no_value=True)
  config.optionxform = str
  config.read(filename)

  for section in ("projet", "directories"):
    if not config.has_section(section):
      print(
        "Missing section {} in {}, we stop".format(section, filename)
      )
      exit(9)

  # ... Project name ...
  # --------------------
  section = "projet"
  option  = "name"
  # project_name = config.get(section, option)

  # ... Common directories ...
  # --------------------------
  section = "directories"
  for option in config.options(section):
    DIR[option] = config.get(section, option)

    if DIR[option] and not os.path.isdir(DIR[option]):
      print("mkdir {}".format(DIR[option]))
      try:
        os.makedirs(DIR[option])
      except Exception as rc:
        print("Could not create {}:\n{}".format(DIR[option], rc))

  # .. DODS configuration ...
  # -------------------------
  section = "dods"
  for option in config.options(section):
    DODS[option] = config.get(section, option)

  return (DIR, DODS)


########################################################################
def string_to_percent(x):
  """
  """
  return float(x.strip("%"))/100.


########################################################################
def string_to_size_unit(x):
  """
  """
  if unicode(x).isdecimal():
    x = x + "o"

  (size, unit) = (float(x[:-1]), x[-1])

  return SizeUnit(size, unit)


########################################################################
def string_to_float(x):
  """
  """
  return float(x.strip("h"))


########################################################################
def string_to_date(ssaammjj, fmt="%Y%m%d"):
  """
  """
  fmts = ["%Y%m%d", "%Y-%m-%d", "%Y_%m_%d"]

  for fmt in fmts:
    try:
      res = dt.datetime.strptime(ssaammjj, fmt)
    except Exception:
      pass
    else:
      break

  return res.date()


########################################################################
def string_to_datetime(string, fmt="%Y-%m-%d-%H:%M"):
  """
  """
  return dt.datetime.strptime(string, fmt)


########################################################################
def where_we_run():

  res = ""
  if "curie" in socket.getfqdn():
    res = "curie"
  elif "ipsl" in socket.getfqdn():
    res = "ipsl"
  else:
    res = "default"

  return res


########################################################################
def get_last_file(dir_data, pattern):
  """
  """
  current_dir = os.getcwd()
  os.chdir(dir_data)
  filename = pattern + "*"
  file_list = glob.glob(os.path.join(dir_data, filename))
  if file_list:
    res = sorted(file_list)[-1]
  else:
    res = None
  os.chdir(current_dir)
  return res


########################################################################
def get_input_files(dir_data, file_list):
  """
  """
  res = []

  for filename in file_list:
    res.append(get_last_file(dir_data, filename))

  if None in res:
    print("\nMissing one or more input files, we stop.")
    for f_in, f_out in zip(file_list, res):
      print("=> {}: {}".format(f_in, f_out))
    exit(9)

  return res


########################################################################
def plot_save(img_out, title, png=False):
  """
  """
  import matplotlib.pyplot as plt
  from matplotlib.backends.backend_pdf import PdfPages

  dpi = 200.

  dirname = os.path.dirname(img_out)
  if not os.path.isdir(dirname):
    print("mkdir {}".format(dirname))
    try:
      os.makedirs(dirname)
    except Exception as rc:
      print("Could not create {}:\n{}".format(dirname, rc))

  if png:
    img_png = img_out.replace(".pdf", ".png")
    plt.savefig(
      img_png,
      dpi=dpi,
      format="png",
      # transparent=True,
      # bbox_inches="tight",
      # pad_inches=0.2 * cm2in,
    )

  with PdfPages(img_out) as pdf:
    pdf.savefig(dpi=dpi)

    # ... pdf file's metadata ...
    # ---------------------------
    d = pdf.infodict()
    d["Title"]   = title
    d["Author"]  = os.path.basename(__file__)

    # d["Subject"] = (
    #   "Time spent over specific commands during create_ts "
    #   "jobs at IDRIS and four configurations at TGCC"
    # )
    # d["Keywords"] = "bench create_ts TGCC IDRIS ncrcat"
    # d["CreationDate"] = dt.datetime(2009, 11, 13)
    # d["ModDate"] = dt.datetime.today()


########################################################################
class AllocItem(object):

  # --------------------------------------------------------------------
  def __init__(
    self,
    alloc_id,
    machine,
    node_type,
    start_date,
    end_date,
    alloc
  ):

    self.alloc_id   = alloc_id
    self.machine    = machine
    self.node_type  = node_type
    self.start_date = start_date
    self.end_date   = end_date
    self.alloc      = alloc

    delta = self.end_date - self.start_date
    self.days = delta.days + 1

    self.daily_conso = self.alloc / self.days

  # --------------------------------------------------------------------
  def __repr__(self):
    return "{} ({:%Y%m%d}/{:%Y%m%d}): {}".format(
      self.machine,
      self.start_date,
      self.end_date,
      self.alloc,
    )


########################################################################
class Project(object):

  # --------------------------------------------------------------------
  def __init__(self, project_name, center):
    self.project = project_name
    self.centre = center
    self.alloc_items = []

  # --------------------------------------------------------------------
  def __repr__(self):
    return "{}/{}: {}".format(
      self.project,
      self.centre,
      self.alloc_items,
    )

  # --------------------------------------------------------------------
  def add_alloc(
    self,
    alloc_id,
    machine,
    node_type,
    start_date,
    end_date,
    alloc
  ):

    alloc_item = AllocItem(
      alloc_id,
      machine,
      node_type,
      start_date,
      end_date,
      alloc
    )

    self.alloc_items.append(alloc_item)

    self.start_date = min(
      [item.start_date for item in self.alloc_items]
    )
    self.end_date = max(
      [item.end_date for item in self.alloc_items]
    )

    self.alloc = sum(
      [item.alloc for item in self.alloc_items]
    )
    self.max_daily_conso = max(
      [item.daily_conso for item in self.alloc_items]
    )

    self.days = sum(
      [item.days for item in self.alloc_items]
    )

    self.nb_alloc = len(self.alloc_items)

  # --------------------------------------------------------------------
  def get_theo_eq(self, dates):

    # x0 = 0
    y0 = 0.

    for item in self.alloc_items:
      yi = y0
      yf = y0 + item.alloc / self.alloc

      if item.start_date.date() in dates:
        xi = dates.index(item.start_date.date())
      else:
        xi = 0

      if item.end_date.date() in dates:
        xf = dates.index(item.end_date.date())
      else:
        xf = len(dates) + 1

      m = np.array([[xi, 1.], [xf+1, 1.]])
      n = np.array([yi, yf])

      try:
        polynome = np.poly1d(np.linalg.solve(m, n))
      except np.linalg.linalg.LinAlgError:
        print("error poly")
        item.theo_eq = None
      else:
        item.theo_eq = polynome
        item.xi = xi
        item.xf = xf
        item.yi = yi
        item.yf = yf

      y0 = yf

  # --------------------------------------------------------------------
  def fill_data(self, row):
    # self.id = row["id"]
    # self.machine = row["machine"]
    # self.node = row["node_type"]
    # self.start_date = row["start_date"]
    # self.end_date = row["end_date"]
    # self.alloc = row["total_hrs"]

    delta = self.end_date - self.start_date
    self.days = delta.days + 1


########################################################################
class SizeUnit(object):
  # --------------------------------------------------------------------
  def __init__(self, size, unit):
    self.size = size
    self.unit = unit

  # --------------------------------------------------------------------
  def __repr__(self):
    return "{:6.2f}{}o".format(self.size, self.unit)

  # --------------------------------------------------------------------
  def convert_size(self, unit_out):
    """
    """
    prefixes = ["o", "K", "M", "G", "T", "P", "H"]

    if not self.size or \
       self.unit == unit_out:
      size_out = self.size
    else:
      idx_deb = prefixes.index(self.unit)
      idx_fin = prefixes.index(unit_out)
      size_out = self.size
      for i in xrange(abs(idx_fin-idx_deb)):
        if idx_fin > idx_deb:
          size_out = size_out / 1024
        else:
          size_out = size_out * 1024

    return SizeUnit(size_out, unit_out)


########################################################################
if __name__ == '__main__':
  pass