source: TOOLS/ConsoGENCMIP6/bin/libconso.py @ 2464

Last change on this file since 2464 was 2464, checked in by labetoulle, 7 years ago
  • Use allow_no_value option from ConfigParser?, for dods directory in config.ini
  • keep default config.ini.init on svn instead of config.ini, which is to be adapted to each implementation
  • debug project_name processing
  • Property svn:executable set to *
File size: 6.9 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# this must come first
5from __future__ import print_function, unicode_literals, division
6
7# standard library imports
8import socket
9import os
10import os.path
11import glob
12import shutil
13import subprocess
14import datetime as dt
15import numpy as np
16import ConfigParser as cp
17
18# Application library imports
19
20
21########################################
22def dods_cp(filein, DIR):
23  """
24  """
25  if not DIR["DODS"]:
26    print("DODS directory not defined")
27    return
28
29  basefile = os.path.basename(filein)
30
31  fileout = os.path.join(DIR["DODS"], basefile)
32  filepng = os.path.join(DIR["DODS"], "img", basefile.split(".")[0] + ".png")
33
34  # Copy file
35  shutil.copy(filein, fileout)
36
37  # Convert it to png for web page
38  command = ["convert", "-density", "200", fileout, filepng]
39
40  try :
41    subprocess.call(command)
42  except Exception as rc :
43    print("Error in convert for {}:\n{}".format(fileout, rc))
44
45  return
46
47
48########################################
49def parse_config(filename):
50
51  DIR = {}
52  OUT = {}
53
54  config = cp.ConfigParser(allow_no_value=True)
55  config.optionxform = str
56  config.read(filename)
57
58  for section in ("projet", "directories", "files"):
59    if not config.has_section(section):
60      print("Missing section {} in {}, we stop".format(section, filename))
61      exit(1)
62
63  # .. Project name ..
64  # ------------------
65  section = "projet"
66  option  = "name"
67  project_name = config.get(section, option)
68
69  # ..Common directories ..
70  # -----------------------
71  section = "directories"
72  for option in config.options(section):
73    DIR[option] = config.get(section, option)
74
75    if DIR[option] and not os.path.isdir(DIR[option]):
76      print("mkdir {}".format(DIR[option]))
77      try :
78        os.makedirs(DIR[option])
79      except Exception as rc :
80        print("Could not create {}:\n{}".format(DIR[option], rc))
81
82  # ..Common files ..
83  # -----------------
84  section = "files"
85  for option in config.options(section):
86    OUT[option] = config.get(section, option)
87
88  return (project_name, DIR, OUT)
89
90
91########################################
92def string_to_percent(x):
93  """
94  """
95  return float(x.strip("%"))/100.
96
97
98########################################
99def string_to_size_unit(x):
100  """
101  """
102  if unicode(x).isdecimal():
103    x = x + "o"
104
105  (size, unit) = (float(x[:-1]), x[-1])
106
107  return SizeUnit(size, unit)
108
109
110########################################
111def string_to_float(x):
112  """
113  """
114  return float(x.strip("h"))
115
116
117########################################
118def string_to_date(ssaammjj, fmt="%Y-%m-%d"):
119  """
120  """
121  return dt.datetime.strptime(ssaammjj, fmt)
122
123
124########################################
125def string_to_datetime(string, fmt="%Y-%m-%d-%H:%M"):
126  """
127  """
128  return dt.datetime.strptime(string, fmt)
129
130
131# ########################################
132# def date_to_string(dtdate, fmt="%Y-%m-%d"):
133#   """
134#   """
135#   return dt.datetime.strftime(dtdate, fmt)
136
137
138########################################
139def where_we_run():
140
141  res = ""
142  if "curie" in socket.getfqdn():
143    res = "curie"
144  elif "ipsl" in socket.getfqdn():
145    res = "ipsl"
146  else:
147    res = "default"
148
149  return res
150
151
152########################################
153def get_last_file(dir_data, pattern):
154  """
155  """
156  current_dir = os.getcwd()
157  os.chdir(dir_data)
158  filename = pattern + "*"
159  file_list = glob.glob(os.path.join(dir_data, filename))
160  if file_list:
161    res = sorted(file_list)[-1]
162  else:
163    res = None
164  os.chdir(current_dir)
165  return res
166
167
168########################################
169def get_input_files(dir_data, file_list):
170  """
171  """
172  res = []
173
174  for filename in file_list:
175    res.append(get_last_file(dir_data, filename))
176
177  if None in res:
178    print("\nMissing one or more input files, we stop.")
179    for f_in, f_out in zip(file_list, res):
180      print("=> {}: {}".format(f_in, f_out))
181    exit(1)
182
183  return res
184
185
186########################################
187def plot_save(img_in, img_out, title, DIR):
188  """
189  """
190  from matplotlib.backends.backend_pdf import PdfPages
191
192  dpi = 200.
193
194  # img_in  = os.path.join(DIR["PLOT"], "{}.pdf".format(img_name))
195
196  with PdfPages(img_in) as pdf:
197    pdf.savefig(dpi=dpi)
198
199    # pdf file's metadata
200    d = pdf.infodict()
201    d["Title"]   = title
202    d["Author"]  = os.path.basename(__file__)
203    # d["Subject"] = "Time spent over specific commands during create_ts \
204    #                 jobs at IDRIS and four configurations at TGCC"
205    # d["Keywords"] = "bench create_ts TGCC IDRIS ncrcat"
206    # d["CreationDate"] = dt.datetime(2009, 11, 13)
207    # d["ModDate"] = dt.datetime.today()
208
209  if os.path.isdir(DIR["SAVEPLOT"]):
210    # img_out = os.path.join(DIR["SAVEPLOT"],
211    #                        "{}_{}.pdf".format(img_name, suffix))
212    shutil.copy(img_in, img_out)
213
214
215########################################
216class Project(object):
217
218  #---------------------------------------
219  def __init__(self, project_name):
220    self.project   = project_name
221    self.date_init = ""
222    self.deadline  = ""
223    self.alloc     = 0
224
225  #---------------------------------------
226  def fill_data(self, filein):
227    import json
228    dico = json.load(open(filein, "r"))
229    self.deadline = string_to_date(dico["deadline"]) + \
230                    dt.timedelta(days=-1)
231    self.alloc = dico["alloc"]
232
233  #---------------------------------------
234  def get_date_init(self, filein):
235    data = np.genfromtxt(
236      filein,
237      skip_header=1,
238      converters={
239        0: string_to_date,
240        1: string_to_percent,
241      },
242      missing_values="nan",
243    )
244    dates, utheos = zip(*data)
245
246    (x1, x2) = (np.nanargmin(utheos), np.nanargmax(utheos))
247
248    m = np.array([[x1, 1.], [x2, 1.]])
249    n = np.array([utheos[x1], utheos[x2]])
250
251    try:
252      (a, b) = np.linalg.solve(m, n)
253    except np.linalg.linalg.LinAlgError:
254      (a, b) = (None, None)
255
256    if a and b:
257      delta = int(round((-b/a)-x1 + 1))
258
259      d1 = dates[x1]
260      self.date_init = d1 + dt.timedelta(days=delta)
261    else:
262      self.date_init = dt.datetime(self.deadline.year, 1, 1)
263
264    delta = self.deadline - self.date_init
265    self.days = delta.days + 1
266
267
268########################################
269class SizeUnit(object):
270  #---------------------------------------
271  def __init__(self, size, unit):
272    self.size = size
273    self.unit = unit
274
275  #---------------------------------------
276  def __repr__(self):
277    return "{:6.2f}{}o".format(self.size, self.unit)
278
279  #---------------------------------------
280  def convert_size(self, unit_out):
281    """
282    """
283    prefixes = ["o", "K", "M", "G", "T", "P", "H"]
284
285    if not self.size or \
286       self.unit == unit_out:
287      size_out = self.size
288    else:
289      idx_deb = prefixes.index(self.unit)
290      idx_fin = prefixes.index(unit_out)
291      size_out = self.size
292      for i in xrange(abs(idx_fin-idx_deb)):
293        if idx_fin > idx_deb:
294          size_out = size_out / 1024
295        else:
296          size_out = size_out * 1024
297
298    return SizeUnit(size_out, unit_out)
299
300
301########################################
302if __name__ == '__main__':
303  pass
304
Note: See TracBrowser for help on using the repository browser.