source: TOOLS/ConsoGENCMIP6/bin/gencmip6.py @ 2437

Last change on this file since 2437 was 2437, checked in by labetoulle, 7 years ago

Add plot_bilan_jobs.py and do some cleaning

  • Property svn:executable set to *
File size: 5.8 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# this must come first
5from __future__ import print_function, unicode_literals, division
6
7# standard library imports
8import socket
9import os
10import os.path
11import glob
12import shutil
13import subprocess
14import datetime as dt
15import numpy as np
16
17# Application library imports
18from gencmip6_path import *
19
20
21########################################
22def dods_cp(filein):
23  """
24  """
25  if not DIR["DODS"]:
26    print("DODS directory not defined")
27    return
28
29  basefile = os.path.basename(filein)
30
31  fileout = os.path.join(DIR["DODS"], basefile)
32  filepng = os.path.join(DIR["DODS"], "img", basefile.split(".")[0] + ".png")
33
34  # Copy file
35  shutil.copy(filein, fileout)
36
37  # Convert it to png for web page
38  command = ["convert", "-density", "200", fileout, filepng]
39
40  try :
41    subprocess.call(command)
42  except Exception as rc :
43    print("Error in convert for {}:\n{}".format(fileout, rc))
44
45  return
46
47
48########################################
49def string_to_percent(x):
50  """
51  """
52  return float(x.strip("%"))/100.
53
54
55########################################
56def string_to_size_unit(x):
57  """
58  """
59  if unicode(x).isdecimal():
60    x = x + "o"
61
62  (size, unit) = (float(x[:-1]), x[-1])
63
64  return SizeUnit(size, unit)
65
66
67########################################
68def string_to_float(x):
69  """
70  """
71  return float(x.strip("h"))
72
73
74########################################
75def string_to_date(ssaammjj, fmt="%Y-%m-%d"):
76  """
77  """
78  return dt.datetime.strptime(ssaammjj, fmt)
79
80
81########################################
82def string_to_datetime(string, fmt="%Y-%m-%d-%H:%M"):
83  """
84  """
85  return dt.datetime.strptime(string, fmt)
86
87
88# ########################################
89# def date_to_string(dtdate, fmt="%Y-%m-%d"):
90#   """
91#   """
92#   return dt.datetime.strftime(dtdate, fmt)
93
94
95########################################
96def where_we_run():
97
98  res = ""
99  if "curie" in socket.getfqdn():
100    res = "curie"
101  elif "ipsl" in socket.getfqdn():
102    res = "ipsl"
103  else:
104    res = "default"
105
106  return res
107
108
109########################################
110def get_last_file(dir_data, pattern):
111  """
112  """
113  current_dir = os.getcwd()
114  os.chdir(dir_data)
115  filename = pattern + "*"
116  file_list = glob.glob(os.path.join(dir_data, filename))
117  if file_list:
118    res = sorted(file_list)[-1]
119  else:
120    res = None
121  os.chdir(current_dir)
122  return res
123
124
125########################################
126def get_input_files(dir_data, file_list):
127  """
128  """
129  res = []
130
131  for filename in file_list:
132    res.append(get_last_file(dir_data, filename))
133
134  if None in res:
135    print("\nMissing one or more input files, we stop.")
136    for f_in, f_out in zip(file_list, res):
137      print("=> {}: {}".format(f_in, f_out))
138    exit(1)
139
140  return res
141
142
143########################################
144def plot_save(img_in, img_out, title):
145  """
146  """
147  from matplotlib.backends.backend_pdf import PdfPages
148
149  dpi = 200.
150
151  # img_in  = os.path.join(DIR["PLOT"], "{}.pdf".format(img_name))
152
153  with PdfPages(img_in) as pdf:
154    pdf.savefig(dpi=dpi)
155
156    # pdf file's metadata
157    d = pdf.infodict()
158    d["Title"]   = title
159    d["Author"]  = os.path.basename(__file__)
160    # d["Subject"] = "Time spent over specific commands during create_ts \
161    #                 jobs at IDRIS and four configurations at TGCC"
162    # d["Keywords"] = "bench create_ts TGCC IDRIS ncrcat"
163    # d["CreationDate"] = dt.datetime(2009, 11, 13)
164    # d["ModDate"] = dt.datetime.today()
165
166  if os.path.isdir(DIR["SAVEPLOT"]):
167    # img_out = os.path.join(DIR["SAVEPLOT"],
168    #                        "{}_{}.pdf".format(img_name, suffix))
169    shutil.copy(img_in, img_out)
170
171
172########################################
173class Project(object):
174
175  #---------------------------------------
176  def __init__(self):
177    self.project   = ""
178    self.date_init = ""
179    self.deadline  = ""
180    self.alloc     = 0
181
182  #---------------------------------------
183  def fill_data(self, filein):
184    import json
185    dico = json.load(open(filein, "r"))
186    self.project = dico["project"]
187    self.deadline = string_to_date(dico["deadline"]) + \
188                    dt.timedelta(days=-1)
189    self.alloc = dico["alloc"]
190
191  #---------------------------------------
192  def get_date_init(self, filein):
193    data = np.genfromtxt(
194      filein,
195      skip_header=1,
196      converters={0: string_to_date,
197                  1: string_to_percent},
198      missing_values="nan",
199    )
200    dates, utheos = zip(*data)
201
202    (x1, x2) = (np.nanargmin(utheos), np.nanargmax(utheos))
203
204    m = np.array([[x1, 1.], [x2, 1.]])
205    n = np.array([utheos[x1], utheos[x2]])
206
207    try:
208      (a, b) = np.linalg.solve(m, n)
209    except np.linalg.linalg.LinAlgError:
210      (a, b) = (None, None)
211
212    if a and b:
213      delta = int(round((-b/a)-x1 + 1))
214
215      d1 = dates[x1]
216      self.date_init = d1 + dt.timedelta(days=delta)
217    else:
218      self.date_init = dt.datetime(self.deadline.year, 1, 1)
219
220    delta = self.deadline - self.date_init
221    self.days = delta.days + 1
222
223
224########################################
225class SizeUnit(object):
226  #---------------------------------------
227  def __init__(self, size, unit):
228    self.size = size
229    self.unit = unit
230
231  #---------------------------------------
232  def __repr__(self):
233    return "{:6.2f}{}o".format(self.size, self.unit)
234
235  #---------------------------------------
236  def convert_size(self, unit_out):
237    """
238    """
239    prefixes = ["o", "K", "M", "G", "T", "P", "H"]
240
241    if not self.size or \
242       self.unit == unit_out:
243      size_out = self.size
244    else:
245      idx_deb = prefixes.index(self.unit)
246      idx_fin = prefixes.index(unit_out)
247      size_out = self.size
248      for i in xrange(abs(idx_fin-idx_deb)):
249        if idx_fin > idx_deb:
250          size_out = size_out / 1024
251        else:
252          size_out = size_out * 1024
253
254    return SizeUnit(size_out, unit_out)
255
256
257########################################
258if __name__ == '__main__':
259  pass
260
Note: See TracBrowser for help on using the repository browser.