Index: TOOLS/ConsoGENCI/trunk/bin/get_project_list.py
===================================================================
--- TOOLS/ConsoGENCI/trunk/bin/get_project_list.py	(revision 3083)
+++ TOOLS/ConsoGENCI/trunk/bin/get_project_list.py	(revision 3083)
@@ -0,0 +1,701 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# ==================================================================== #
+# Author: Sonia Labetoulle                                             #
+# Contact: sonia.labetoulle _at_ ipsl.jussieu.fr                       #
+# Created: 2016                                                        #
+# History:                                                             #
+# Modification:                                                        #
+# ==================================================================== #
+
+# This must come first
+from __future__ import print_function, unicode_literals, division
+
+# Standard library imports
+import os
+# import math
+from argparse import ArgumentParser
+import shutil
+import pprint
+import collections
+import ConfigParser as cp
+import datetime as dt
+import subprocess
+from distutils.dir_util import copy_tree
+
+# Application library imports
+import libconso_db as cdb
+# import libconso_cpt as ccpt
+import db_data
+
+pp = pprint.PrettyPrinter(indent=2)
+
+
+#######################################################################
+def get_arguments():
+  parser = ArgumentParser()
+  # parser.add_argument("project", action="store",
+  #                     help="Project name")
+  # parser.add_argument("center", action="store",
+  #                     help="Center name (idris/tgcc)")
+
+  parser.add_argument("-v", "--verbose", action="store_true",
+                      help="verbose mode")
+  parser.add_argument("-d", "--dryrun", action="store_true",
+                      help="only print what is to be done")
+  # parser.add_argument("-r", "--range", action="store", nargs=2,
+  #                     help="date range: ssaammjj ssaammjj")
+  return parser.parse_args()
+
+
+########################################################################
+def html_header(project, css_file):
+  html = (
+    "<!DOCTYPE html>\n"
+    "<html lang=\"fr-FR\">\n"
+    "<head>\n"
+    "  <meta charset=\"UTF-8\">\n"
+    "  <meta name=\"description\" content=\"Suivi de consommation {pj}\">\n"
+    "  <meta name=\"author\" content=\"Sonia Labetoulle\">\n"
+    "  <title>Suivi de consommation {pj}</title>\n"
+    "  <link rel=\"stylesheet\" type=\"text/css\" "
+    "href=\"{css}\">\n"
+    "</head>\n"
+    "\n"
+    "<body>\n"
+    "  <div id=\"main\">\n"
+    "\n".format(pj=project.upper(), css=css_file)
+  )
+
+  return html
+
+
+########################################################################
+def html_footer():
+  html = (
+    "\n"
+    "    <div class=\"clear\"></div>\n"
+    "  </div>\n"
+    "</body>\n"
+    "</html>\n"
+  )
+
+  return html
+
+
+########################################################################
+def html_main_index():
+  html = (
+    "    <h1>Conso GENCI</h1>\n"
+  )
+
+  return html
+
+
+########################################################################
+def html_main_project(centre, project, node_subp):
+
+  html = (
+    "    <h1>Conso {pj} ({ce})</h1>\n\n"
+    .format(ce=centre.upper(), pj=project.upper())
+  )
+
+  for (node_type, sub_project) in node_subp:
+    if sub_project:
+      title = "Sous-imputation {sp} / {nd}".format(
+        sp=sub_project if sub_project else project,
+        nd=node_type.replace("/", "")
+      )
+    else:
+      title = "Conso globale {nd}".format(
+        nd=node_type.replace("/", "")
+      )
+    html = html + (
+      "    <div class=\"img\">\n"
+      "      <h2>{tl}</h2>\n"
+      "      <a href=\"../img/bilan_{ce}_{pj}_{sp}_{nd}.png\">\n"
+      "        <img class=\"data\"\n"
+      "             src=\"../img/bilan_{ce}_{pj}_{sp}_{nd}.png\"\n"
+      "             alt=\"{tl}\" />\n"
+      "      </a><br />\n"
+      "      <div class=\"lien\">\n"
+      "        <a href=\"../pdf/bilan_{ce}_{pj}_{sp}_{nd}.pdf\">\n"
+      "          <!--<img class=\"icon\" src=\"utils/pdf_2.jpg\" />-->\n"
+      "          Fichier pdf\n"
+      "        </a>\n"
+      "      </div>\n"
+      "    </div>\n".format(
+        tl=title,
+        ce=centre,
+        pj=project,
+        sp=sub_project if sub_project else project,
+        nd=node_type.replace("/", "")
+      )
+    )
+
+  return html
+
+
+########################################################################
+def html_project(centre, project, node_subp, project_file):
+
+  if args.verbose:
+    print("Create {}".format(project_file))
+
+  header = html_header(project, os.path.join("..", css_file))
+  footer = html_footer()
+  main = html_main_project(centre, project, node_subp)
+
+  subdir = os.path.dirname(project_file)
+  if not os.path.isdir(subdir):
+    try:
+      os.mkdir(subdir)
+    except Exception as rc:
+      raise("Could not create directory {}:\n{}".format(subdir, rc))
+
+  with open(project_file, "w") as fileout:
+    try:
+      fileout.write(header + main + footer)
+    except Exception as rc:
+      raise("Could not create directory {}:\n{}".format(project_file, rc))
+
+
+########################################################################
+def config_write(centre, project):
+  config_file = os.path.join(
+    CARD_DIR,
+    "config_{}_{}.ini".format(centre, project)
+  )
+
+  if not os.path.isfile(config_file):
+    if args.verbose:
+      print("Create {}".format(config_file))
+
+    config = cp.ConfigParser(allow_no_value=True)
+    config.optionxform = str
+
+    config.add_section("projet")
+    config.set("projet", "name", project)
+
+    config.add_section("directories")
+    # config.set("directories", "ROOT_DIR", "/home/igcmg/ConsoGENCI")
+    config.set("directories", "ROOT_DIR", ROOT_DIR)
+    config.set("directories", "DATA", "%(ROOT_DIR)s/data")
+    config.set("directories", "PLOT", "%(ROOT_DIR)s/plot")
+    config.set("directories", "DODS")
+
+    config.add_section("dods")
+    config.set("dods", "DIR", "/home/igcmg/dods/ConsoGENCI")
+    config.set("dods", "USER", "igcmg")
+    config.set("dods", "SERVER", "ciclad")
+
+    with open(config_file, "w") as fileout:
+      try:
+        config.write(fileout)
+      except Exception as rc:
+        raise("Could not write {}:\n{}".format(config_file, rc))
+
+
+#######################################################################
+if __name__ == "__main__":
+
+  date_deb = dt.datetime.now()
+
+  # .. Initialization ..
+  # ====================
+  # ... Command line arguments ...
+  # ------------------------------
+  args = get_arguments()
+  if args.verbose:
+    print(args)
+
+  # ... Files and directories ...
+  # -----------------------------
+  ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+  CARD_DIR = os.path.join(ROOT_DIR, "card")
+  DATA_DIR = os.path.join(ROOT_DIR, "data")
+  PLOT_DIR = os.path.join(ROOT_DIR, "plot")
+  HTML_DIR = os.path.join(ROOT_DIR, "web")
+  DODS_DIR = os.path.join(
+    os.path.dirname(ROOT_DIR),
+    "dods",
+    "ConsoGENCI",
+  )
+  SUBMIT_DIR = os.getcwd()
+  index_file = os.path.join(
+    HTML_DIR,
+    "index_full.html"
+  )
+  # css_file = os.path.join(
+  #   HTML_DIR,
+  #   "utils",
+  #   "ConsoGENCI.css"
+  # )
+  css_file = os.path.join(
+    "utils",
+    "ConsoGENCI.css"
+  )
+
+  # pattern = ccpt.cpt_pattern(center=args.center, project=args.project)
+  # dirin = os.path.join(DATA_DIR, args.center, "tmp")
+  # dirout = os.path.join(DATA_DIR, args.center, args.project)
+  # filelist = ccpt.find_input_files(dirin, pattern, args.range)
+
+  # if not filelist:
+  #   print("No input files found, exit")
+  #   exit(1)
+
+  # if args.verbose:
+  #   print("SUBMIT_DIR:", SUBMIT_DIR)
+  #   print("DATA_DIR:", DATA_DIR)
+  #   print("dirin:", dirin)
+
+  # .. Connection to database ..
+  # ============================
+  if args.verbose:
+    print("Connection to database")
+  conn, cursor = cdb.connect_db(
+    db_data.db_host,
+    db_data.db_name,
+    db_data.db_user,
+    db_data.db_pwd,
+  )
+
+  # print(DODS_DIR)
+
+  # exit()
+
+  # .. Extract centre/project list from table ..
+  # ============================================
+  table_name = "conso.tbl_allocation"
+  request = (
+    "SELECT DISTINCT A.centre, A.project, A.node_type, C.sub_project "
+    "FROM conso.tbl_allocation AS A "
+    "INNER JOIN conso.tbl_consumption AS C "
+    "ON A.id = C.allocation_id "
+    "WHERE A.is_active = TRUE "
+    "ORDER BY A.centre, A.project, A.node_type, C.sub_project ;"
+  )
+
+  cdb.select_db(cursor, request)
+  # print(cursor.rowcount)
+  # print(cursor.fetchall())
+  result = list(cursor)
+
+  # .. Close connection to database ..
+  # ==================================
+  if args.verbose:
+    print("Close connection")
+  cdb.close_db(conn)
+
+  # .. Build data dictionary ..
+  # ===========================
+  centre_dict = collections.OrderedDict()
+  # for centre, project, node_type, sub_project in result:
+  #   if centre not in centre_dict:
+  #     centre_dict[centre] = collections.OrderedDict()
+  #   if project not in centre_dict[centre]:
+  #     centre_dict[centre][project] = []
+  #   if node_type not in centre_dict[centre][project]:
+  #     centre_dict[centre][project].append(node_type)
+
+  for centre, project, node_type, sub_project in result:
+    if centre not in centre_dict:
+      centre_dict[centre] = collections.OrderedDict()
+    if project not in centre_dict[centre]:
+      centre_dict[centre][project] = []
+    if (node_type, sub_project) not in centre_dict[centre][project]:
+      centre_dict[centre][project].append((node_type, sub_project))
+
+  pp.pprint(centre_dict)
+
+  # .. Data for html index file ..
+  # ==============================
+  idx_header = html_header("GENCI", css_file)
+  idx_footer = html_footer()
+  idx_main = html_main_index()
+
+  # .. Loop over centres ..
+  # =======================
+  for centre, projects in centre_dict.iteritems():
+    print("\n" + centre + "\n=====")
+
+    # ... Add centre to index page ...
+    # --------------------------------
+    idx_main = idx_main + (
+      "\n"
+      "    <div class=\"img\">\n"
+      "      <table>\n"
+      "        <tr>\n"
+      "          <th>{}</th>\n"
+      "        </tr>\n"
+      .format(centre.upper())
+    )
+
+    # ... Loop over projects ...
+    # --------------------------
+    for project, node_subp in projects.iteritems():
+      print(project + "\n--------")
+
+      # Build project card if it doesn't already exist
+      config_write(centre, project)
+
+      # Build html page for project
+      # subdir = os.path.join(
+      #   HTML_DIR,
+      #   centre,
+      # )
+
+      project_file = os.path.join(
+        # subdir,
+        "html",
+        "{}_{}.html".format(centre, project)
+      )
+
+      # if not os.path.isfile(project_file):
+      html_project(
+        centre,
+        project,
+        node_subp,
+        os.path.join("web", project_file),
+      )
+
+      # Add project to index page
+      idx_main = idx_main + (
+        "        <tr>\n"
+        "          <td>\n"
+        "            <a href=\"{}\">\n"
+        "              {}\n"
+        "            </a>\n"
+        "          </td>\n"
+        "        </tr>\n"
+        .format(project_file, project.upper())
+      )
+
+      # Loop over node types
+      for (node_type, sub_project) in node_subp:
+        print("- {} / {}".format(node_type, sub_project))
+
+        # ${ROOT_DIR}/bin/plot_bilan.py -vf --png  gencmip6 tgcc thin/standard
+        #                               --subproject dcpcmip6
+        command = [
+          os.path.join(ROOT_DIR, "bin", "plot_bilan.py"),
+          "-vf",
+          "--png",
+          project,
+          centre,
+          node_type,
+        ]
+        if sub_project:
+          command.extend(["--subproject", sub_project])
+
+        print(command)
+        if not args.dryrun:
+          try:
+            subprocess.call(command)
+          except Exception as rc:
+            print("Error in plot for {}:\n{}".format(command, rc))
+        else:
+          print(command)
+
+        # Copy today's file in web directory
+        filein = "bilan_{ce}_{pj}_{sp}_{nd}_{dt:%Y%m%d}".format(
+          ce=centre,
+          pj=project,
+          sp=sub_project if sub_project else project,
+          nd=node_type.replace("/", ""),
+          dt=dt.datetime.today(),
+        )
+        fileout = "bilan_{ce}_{pj}_{sp}_{nd}".format(
+          ce=centre,
+          pj=project,
+          sp=sub_project if sub_project else project,
+          nd=node_type.replace("/", ""),
+        )
+
+        dirin = os.path.join(
+          PLOT_DIR,
+          centre,
+          project,
+        )
+        pdfout = os.path.join(
+          HTML_DIR,
+          # centre,
+          "pdf",
+        )
+        pngout = os.path.join(
+          HTML_DIR,
+          # centre,
+          "img",
+        )
+
+        dirnames = [pdfout, pngout]
+        extnames = ["pdf", "png"]
+
+        for dirname, extname in zip(dirnames, extnames):
+          if not os.path.isdir(dirname):
+            print("mkdir {}".format(dirname))
+            try:
+              os.makedirs(dirname)
+            except Exception as rc:
+              print("Could not create {}:\n{}".format(dirname, rc))
+
+          if not args.dryrun:
+            try:
+              shutil.copy(
+                os.path.join(
+                  dirin,
+                  "{}.{}".format(filein, extname),
+                ),
+                os.path.join(
+                  dirname,
+                  "{}.{}".format(fileout, extname),
+                ),
+              )
+            except Exception as rc:
+              print("Could not copy {}.{}:\n{}".format(filein, extname, rc))
+          else:
+            print(
+              "{} => {}".format(
+                os.path.join(
+                  dirin,
+                  "{}.{}".format(filein, extname),
+                ),
+                os.path.join(
+                  dirname,
+                  "{}.{}".format(fileout, extname),
+                ),
+              )
+            )
+
+        # shutil.copy(
+        #   os.path.join(dirin, filein + ".pdf"),
+        #   os.path.join(pdfout, fileout + ".pdf")
+        # )
+        # shutil.copy(
+        #   os.path.join(dirin, filein + ".png"),
+        #   os.path.join(pngout, fileout + ".png")
+        # )
+
+    # ... Close list for current centre ...
+    # -------------------------------------
+    idx_main = idx_main + (
+      "      </table>\n"
+      "    </div>\n"
+    )
+
+  # .. Write index file ..
+  # ======================
+  with open(index_file, "w") as fileout:
+    fileout.write(idx_header + idx_main + idx_footer)
+
+  date_fin = dt.datetime.now()
+  print(date_fin - date_deb)
+
+  # .. Copy everything to dods ..
+  # =============================
+  print("Copy everything to dods")
+  if not args.dryrun:
+    try:
+      res = copy_tree(
+        HTML_DIR,
+        DODS_DIR,
+        preserve_mode=1,
+        preserve_times=1,
+        update=1
+      )
+    except Exception as rc:
+      print("Error while copying to dods:\n{}".format(rc))
+  else:
+    print("{} => {}".format(HTML_DIR, DODS_DIR))
+
+  if args.verbose:
+    print(res)
+
+  exit()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  for key in project_dict:
+    project_dict[key] = [p for c, p in result if c == key]
+  # .. Loop over centres ..
+  # =======================
+  for centre, project_list in project_dict.iteritems():
+    print("\n" + centre)
+
+    # ... Add centre to index page ...
+    # --------------------------------
+    idx_main = idx_main + (
+      '\n'
+      '    <h2>{}</h2>\n'
+      '    <ul>\n'.format(centre.upper())
+    )
+
+    # ... Loop over projects ...
+    # --------------------------
+    for project in project_list:
+      print(project)
+
+      # Build project card if it doesn't already exist
+      config_write(centre, project)
+
+      # Build html page for project
+      subdir = os.path.join(
+        HTML_DIR,
+        centre,
+      )
+
+      project_file = os.path.join(
+        subdir,
+        "{}_{}.html".format(centre, project)
+      )
+
+      if not os.path.isfile(project_file):
+        html_project(centre, project, project_file)
+
+      # Add project to index page
+      idx_main = idx_main + (
+        '      <li>\n'
+        '        <a href="{}">{}</a>\n'
+        '      </li>\n'.format(
+          project_file, project.upper()
+        )
+      )
+
+    # ... Close list for current centre ...
+    # -------------------------------------
+    idx_main = idx_main + '    </ul>\n'
+
+  # .. Write index file ..
+  # ======================
+  with open(index_file, "w") as fileout:
+    fileout.write(idx_header + idx_main + idx_footer)
+
+  # .. Launch plot jobs ..
+  # ======================
+
+  # allocs = []
+  # for row in cursor:
+  #   allocs.append(ccpt.AllocRow(row))
+
+  # # .. Process files ..
+  # # ===================
+  # for filename in filelist:
+
+  #   fileout = os.path.join(dirout, os.path.basename(filename))
+  #   if os.path.isfile(fileout):
+  #     print(
+  #       "file {} already processed, remove file".format(
+  #         os.path.basename(filename)
+  #       )
+  #     )
+  #     if not args.dryrun:
+  #       try:
+  #         os.remove(filename)
+  #       except Exception as rc:
+  #         print("Could not move {}:\n{}".format(filename, rc))
+  #     break
+
+  #   # .. Build dictionary from files ..
+  #   # =================================
+  #   conso_per_alloc = {}
+
+  #   date, blocs = ccpt.parse_input_cpt(
+  #     filename, args.project, args.center, mode_conso=True
+  #   )
+
+  #   for bloc in blocs:
+  #     if not bloc.alloc_id:
+  #       bloc.alloc_id = ccpt.get_project_id(bloc, allocs)
+  #       if not bloc.alloc_id:
+  #         print("no alloc id found, skip bloc")
+  #         print(bloc.machine, bloc.node, bloc.cpt_date)
+  #         pp.pprint(allocs)
+  #         continue
+  #     if bloc.alloc_id not in conso_per_alloc:
+  #       conso_per_alloc[bloc.alloc_id] = set()
+  #     for conso in bloc.consos:
+  #       conso_per_alloc[bloc.alloc_id].add(conso)
+
+  #   # .. Insert data in table, one alloc_id at a time ..
+  #   # ==================================================
+  #   for alloc_id, consos in conso_per_alloc.iteritems():
+  #     # ... Create request sub string ...
+  #     # ---------------------------------
+  #     lines_req = [
+  #       (
+  #         "('{alloc}', "
+  #         "'{date}', "
+  #         "{total_hrs}, "
+  #         "{login}, "
+  #         "{create})"
+  #       ) .format(
+  #         alloc=alloc_id,
+  #         date=item.date,
+  #         total_hrs=item.conso if not math.isnan(item.conso) else "'NaN'",
+  #         login="'"+item.login+"'" if item.login != "total" else "NULL",
+  #         create="CURRENT_TIMESTAMP",
+  #       ) for item in consos
+  #     ]
+
+  #     # ... Create full request ...
+  #     # ---------------------------
+  #     table_name = "conso.tbl_consumption"
+  #     request = (
+  #       "INSERT INTO " + table_name + " ("
+  #       "  allocation_id, "
+  #       "  date, "
+  #       "  total_hrs, "
+  #       "  login, "
+  #       "  row_create_date "
+  #       ") "
+  #       "VALUES "
+  #     )
+  #     request = request + ", ".join(lines_req)
+
+  #     # ... Execute request ...
+  #     # -----------------------
+  #     if args.verbose:
+  #       print("Execute request for alloc_id = {}".format(alloc_id))
+  #       print(request)
+  #     cdb.insert_db(cursor, request)
+
+  #     # ... Commit inserts ...
+  #     # ----------------------
+  #     if not args.dryrun:
+  #       if args.verbose:
+  #         print("Commit inserts")
+  #       cdb.commit_db(conn)
+
+  #   # .. Move processed file to project directory ..
+  #   # ===============================================
+  #   if args.verbose:
+  #     print("Move processed file:")
+  #     print("{} => {}".format(filename, dirout))
+  #   if not args.dryrun:
+  #     try:
+  #       shutil.move(filename, dirout)
+  #     except Exception as rc:
+  #       print("Could not move {}:\n{}".format(filename, rc))
+
+  exit(0)
Index: TOOLS/ConsoGENCI/trunk/bin/insert_conso_tbl.py
===================================================================
--- TOOLS/ConsoGENCI/trunk/bin/insert_conso_tbl.py	(revision 2783)
+++ TOOLS/ConsoGENCI/trunk/bin/insert_conso_tbl.py	(revision 3083)
@@ -68,5 +68,5 @@
 
   if not filelist:
-    print("No files found, exit")
+    print("No input files found, exit")
     exit(1)
 
Index: TOOLS/ConsoGENCI/trunk/bin/libconso.py
===================================================================
--- TOOLS/ConsoGENCI/trunk/bin/libconso.py	(revision 2783)
+++ TOOLS/ConsoGENCI/trunk/bin/libconso.py	(revision 3083)
@@ -11,5 +11,6 @@
 
 # this must come first
-from __future__ import print_function, unicode_literals, division
+# from __future__ import print_function, unicode_literals, division
+from __future__ import print_function, division
 
 # standard library imports
@@ -25,4 +26,10 @@
 # Application library imports
 
+########################################################################
+
+# .. Constants and variables ..
+# =============================
+cm2in = 1. / 2.54
+
 
 ########################################################################
@@ -34,6 +41,24 @@
     return
 
-  # .. Copy pdf file to dods server ..
+  # .. Convert pdf to temporary png ..
   # ==================================
+  import matplotlib.pyplot as plt
+
+  img_png = img_out.replace(".pdf", "_tmp.png")
+  dpi = 200.
+
+  plt.savefig(
+    img_png,
+    dpi=dpi,
+    format="png",
+    # transparent=True,
+    # bbox_inches="tight",
+    # pad_inches=0.2 * cm2in,
+  )
+
+  # .. Copy files to dods server ..
+  # ===============================
+  # ... pdf file ...
+  # ----------------
   command = [
     "scp",
@@ -46,22 +71,11 @@
   ]
   print(command)
-  try :
+  try:
     subprocess.call(command)
-  except Exception as rc :
+  except Exception as rc:
     print("Error in scp for {}:\n{}".format(command, rc))
 
-  # .. Convert pdf to temporary png ..
-  # ==================================
-  img_png = img_out.replace(".pdf", ".png")
-  command = ["convert", "-density", "200", img_out, img_png]
-
-  print(command)
-  try :
-    subprocess.call(command)
-  except Exception as rc :
-    print("Error in convert for {}:\n{}".format(img_out, rc))
-
-  # .. Copy png file to dods server ..
-  # ==================================
+  # ... png file ...
+  # ----------------
   command = [
     "scp",
@@ -75,7 +89,7 @@
 
   print(command)
-  try :
+  try:
     subprocess.call(command)
-  except Exception as rc :
+  except Exception as rc:
     print("Error in scp for {}:\n{}".format(command, rc))
 
@@ -121,7 +135,7 @@
     if DIR[option] and not os.path.isdir(DIR[option]):
       print("mkdir {}".format(DIR[option]))
-      try :
+      try:
         os.makedirs(DIR[option])
-      except Exception as rc :
+      except Exception as rc:
         print("Could not create {}:\n{}".format(DIR[option], rc))
 
@@ -234,7 +248,8 @@
 
 ########################################################################
-def plot_save(img_out, title):
-  """
-  """
+def plot_save(img_out, title, png=False):
+  """
+  """
+  import matplotlib.pyplot as plt
   from matplotlib.backends.backend_pdf import PdfPages
 
@@ -244,8 +259,19 @@
   if not os.path.isdir(dirname):
     print("mkdir {}".format(dirname))
-    try :
+    try:
       os.makedirs(dirname)
-    except Exception as rc :
+    except Exception as rc:
       print("Could not create {}:\n{}".format(dirname, rc))
+
+  if png:
+    img_png = img_out.replace(".pdf", ".png")
+    plt.savefig(
+      img_png,
+      dpi=dpi,
+      format="png",
+      # transparent=True,
+      # bbox_inches="tight",
+      # pad_inches=0.2 * cm2in,
+    )
 
   with PdfPages(img_out) as pdf:
@@ -270,5 +296,5 @@
 class AllocItem(object):
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def __init__(
     self,
@@ -293,5 +319,5 @@
     self.daily_conso = self.alloc / self.days
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def __repr__(self):
     return "{} ({:%Y%m%d}/{:%Y%m%d}): {}".format(
@@ -306,5 +332,5 @@
 class Project(object):
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def __init__(self, project_name, center):
     self.project = project_name
@@ -312,5 +338,5 @@
     self.alloc_items = []
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def __repr__(self):
     return "{}/{}: {}".format(
@@ -320,5 +346,5 @@
     )
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def add_alloc(
     self,
@@ -362,5 +388,5 @@
     self.nb_alloc = len(self.alloc_items)
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def get_theo_eq(self, dates):
 
@@ -399,5 +425,5 @@
       y0 = yf
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def fill_data(self, row):
     # self.id = row["id"]
@@ -414,14 +440,14 @@
 ########################################################################
 class SizeUnit(object):
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def __init__(self, size, unit):
     self.size = size
     self.unit = unit
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def __repr__(self):
     return "{:6.2f}{}o".format(self.size, self.unit)
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def convert_size(self, unit_out):
     """
Index: TOOLS/ConsoGENCI/trunk/bin/libconso_cpt.py
===================================================================
--- TOOLS/ConsoGENCI/trunk/bin/libconso_cpt.py	(revision 2783)
+++ TOOLS/ConsoGENCI/trunk/bin/libconso_cpt.py	(revision 3083)
@@ -19,5 +19,6 @@
 
 # This must come first
-from __future__ import print_function, unicode_literals, division
+# from __future__ import print_function, unicode_literals, division
+from __future__ import print_function, division
 
 # Standard library imports
Index: TOOLS/ConsoGENCI/trunk/bin/libconso_db.py
===================================================================
--- TOOLS/ConsoGENCI/trunk/bin/libconso_db.py	(revision 2783)
+++ TOOLS/ConsoGENCI/trunk/bin/libconso_db.py	(revision 3083)
@@ -19,5 +19,6 @@
 
 # This must come first
-from __future__ import print_function, unicode_literals, division
+# from __future__ import print_function, unicode_literals, division
+from __future__ import print_function, division
 
 # Standard library imports
@@ -35,10 +36,13 @@
   )
 
-  # # Print the connection string we will use to connect
-  # print("Connecting to database\n  -> {}".format(conn_string))
+  # Print the connection string we will use to connect
+  print("Connecting to database\n  -> {}".format(
+      conn_string.replace(db_pwd, "*****")
+    )
+  )
 
   # Get a connection, if a connect cannot be made an exception will be
   # raised here
-  try :
+  try:
     conn = psycopg2.connect(conn_string)
   except Exception as rc:
Index: TOOLS/ConsoGENCI/trunk/bin/plot_bilan.py
===================================================================
--- TOOLS/ConsoGENCI/trunk/bin/plot_bilan.py	(revision 2783)
+++ TOOLS/ConsoGENCI/trunk/bin/plot_bilan.py	(revision 3083)
@@ -10,6 +10,15 @@
 # ==================================================================== #
 
+# ==================================================================== #
+# ssh readonly@prodiguer-test-db.ipsl.upmc.fr                          #
+# psql -U prodiguer_db_user prodiguer                                  #
+#                                                                      #
+# ssh readonly@prodiguer-test-db.ipsl.upmc.fr -L 5432:localhost:5432   #
+# psql -h prodiguer-test-db.ipsl.upmc.fr -U prodiguer_db_user prodiguer
+# ==================================================================== #
+
 # this must come first
 from __future__ import print_function, unicode_literals, division
+# from __future__ import print_function, division
 
 # standard library imports
@@ -26,5 +35,5 @@
 ########################################################################
 class PlotData(object):
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def __init__(self, date_min, date_max):
     self.date_min = date_min
@@ -37,9 +46,9 @@
 ########################################################################
 class DataDict(dict):
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def __init__(self):
     self = {}
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def init_range(self, date_beg, date_end, inc=1):
     """
@@ -55,5 +64,5 @@
       self.add_item(date)
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def fill_data(self, projet):
     """
@@ -71,4 +80,9 @@
     )
 
+    if args.subproject:
+      subp_condition = "sub_project = \'{}\'".format(args.subproject)
+    else:
+      subp_condition = "sub_project IS NULL"
+
     table_name = "conso.tbl_consumption"
     request = (
@@ -76,4 +90,6 @@
       "FROM " + table_name + " "
       "WHERE login IS NULL "
+      # "  AND sub_project IS NULL "
+      "  AND " + subp_condition + " "
       "  AND " + id_condition + " "
       "ORDER BY date"
@@ -83,5 +99,4 @@
     if args.verbose:
       print("Access table \"{}\"".format(table_name))
-      print(request)
     cdb.select_db(cursor, request)
 
@@ -96,5 +111,5 @@
         self[date.date()].fill()
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def add_item(self, date, conso=np.nan,
                real_use=np.nan, theo_use=np.nan,
@@ -111,5 +126,5 @@
     )
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def get_items_in_range(self, date_beg, date_end, inc=1):
     """
@@ -122,5 +137,5 @@
     return items[::inc]
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def get_items_in_full_range(self, inc=1):
     """
@@ -132,5 +147,5 @@
     return items[::inc]
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def get_items(self, inc=1):
     """
@@ -144,30 +159,30 @@
 
 class Conso(object):
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def __init__(self, date, conso=np.nan,
                real_use=np.nan, theo_use=np.nan,
                run_mean=np.nan, pen_mean=np.nan,
                run_std=np.nan, pen_std=np.nan):
-    self.date     = date
-    self.conso    = conso
-    self.real_use = real_use
-    self.theo_use = theo_use
-    self.poly_theo = np.poly1d([])
+    self.date       = date
+    self.conso      = conso
+    self.real_use   = real_use
+    self.theo_use   = theo_use
+    self.poly_theo  = np.poly1d([])
     self.poly_delay = np.poly1d([])
-    self.run_mean = run_mean
-    self.pen_mean = pen_mean
-    self.run_std  = run_std
-    self.pen_std  = pen_std
-    self.filled   = False
-
-  #---------------------------------------------------------------------
+    self.run_mean   = run_mean
+    self.pen_mean   = pen_mean
+    self.run_std    = run_std
+    self.pen_std    = pen_std
+    self.filled     = False
+
+  # --------------------------------------------------------------------
   def __repr__(self):
     return "{:.2f} ({:.2%})".format(self.conso, self.real_use)
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def isfilled(self):
     return self.filled
 
-  #---------------------------------------------------------------------
+  # --------------------------------------------------------------------
   def fill(self):
     self.filled = True
@@ -177,5 +192,6 @@
 def plot_init():
   paper_size  = np.array([29.7, 21.0])
-  fig, ax_conso = plt.subplots(figsize=(paper_size/2.54))
+  fig, ax_conso = plt.subplots(figsize=(paper_size * cm2in))
+  # fig, ax_conso = plt.subplots(figsize=(paper_size/2.54))
   ax_theo = ax_conso.twinx()
 
@@ -329,5 +345,5 @@
     if d.weekday() == 0:
       ax_conso.axvline(
-        x=x, color="black", alpha=0.5,
+        x=x, color="gray", alpha=0.5,
         linewidth=0.5, linestyle=":"
       )
@@ -393,6 +409,16 @@
   parser.add_argument("project", action="store",
                       help="Project name")
-  parser.add_argument("center", action="store",
-                      help="Center name (idris/tgcc)")
+  parser.add_argument("centre", action="store",
+                      choices=["idris", "tgcc"],
+                      help="Centre name")
+  parser.add_argument(
+    "node", action="store", default="thin/standard",
+    choices=["thin/standard", "standard", "fat/large", "hybrid"],
+    help="Node type"
+  )
+
+  parser.add_argument("--subproject", action="store",
+                      type=str, default=None,
+                      help="Sub-project")
 
   parser.add_argument("-v", "--verbose", action="store_true",
@@ -400,4 +426,6 @@
   parser.add_argument("-f", "--full", action="store_true",
                       help="plot the whole period")
+  parser.add_argument("--png", action="store_true",
+                      help="pgn output in addition to pdf")
   parser.add_argument("-i", "--increment", action="store",
                       type=int, default=1, dest="inc",
@@ -448,5 +476,5 @@
   config_file = os.path.join(
     "card",
-    "config_{}_{}.ini".format(args.center, args.project)
+    "config_{}_{}.ini".format(args.centre, args.project)
   )
 
@@ -461,10 +489,12 @@
   #                     [OUT["PARAM"], OUT["UTHEO"], OUT["BILAN"]])
 
-  img_name = "{}_{}_{}".format(
+  img_name = "{}_{}_{}_{}_{}".format(
     os.path.splitext(
       os.path.basename(__file__)
     )[0].replace("plot_", ""),
+    args.centre,
     args.project,
-    args.center,
+    args.subproject if args.subproject else args.project,
+    args.node.replace("/", ""),
   )
 
@@ -509,4 +539,6 @@
     "FROM " + table_name + " "
     "WHERE project = '" + args.project + "' "
+    "AND node_type = '" + args.node + "' "
+    "AND is_active = TRUE "
     "ORDER BY start_date"
     ";"
@@ -518,5 +550,5 @@
   # print(cursor.description)
 
-  projet = Project(args.project, args.center)
+  projet = Project(args.project, args.centre)
   for row in cursor:
     if (bilan_plot.date_min >= row["start_date"].date() and
@@ -572,6 +604,8 @@
   projet.get_theo_eq(dates)
 
-  cumul = np.array([item.conso for item in selected_items],
-                        dtype=float)
+  cumul = np.array(
+    [item.conso for item in selected_items],
+    dtype=float
+  )
   consos = []
   consos.append(cumul[0])
@@ -595,5 +629,10 @@
     theo_uses.append(100. * poly_theo(dates.index(date)))
     real_uses2.append((100 * cumul[idx]) / alloc_val)
-  delay_uses = delay * [0., ] + theo_uses[:-delay]
+
+  if nb_items > delay:
+    delay_uses = delay * [0., ] + theo_uses[:-delay]
+  else:
+    delay_uses = nb_items * [0., ]
+
   theo_uses = np.array(theo_uses, dtype=float)
   delay_uses = np.array(delay_uses, dtype=float)
@@ -632,6 +671,8 @@
   # ... Tweak figure ...
   # --------------------
-  title = "Consommation {})".format(
-    projet.project.upper(),
+  proj_title = args.subproject if args.subproject else projet.project
+  title = "Consommation {}".format(
+    # projet.project.upper(),
+    proj_title.upper()
   )
   faitle = "(le {:%d/%m/%Y})".format(
@@ -649,16 +690,23 @@
   if args.range:
     img_out = os.path.join(
-      DIR["PLOT"], args.center, args.project,
+      DIR["PLOT"], args.centre, args.project,
       "{}_{:%Y%m%d}_{:%Y%m%d}.pdf".format(img_name, *args.range)
     )
   else:
     img_out = os.path.join(
-      DIR["PLOT"], args.center, args.project,
+      DIR["PLOT"], args.centre, args.project,
       "{}_{:%Y%m%d}.pdf".format(img_name, today)
     )
   if args.verbose:
     print("Save image as {}".format(img_out))
-
-  plot_save(img_out, title)
+    if args.png:
+      print("Produce png")
+
+  title = "Consommation {} ({}) au {:%Y%m%d}".format(
+    args.project.upper(),
+    args.centre.upper(),
+    today
+  )
+  plot_save(img_out, title, png=args.png)
 
   # ... Publish figure on dods ...
Index: TOOLS/ConsoGENCI/trunk/card/config_center_project.ini.init
===================================================================
--- TOOLS/ConsoGENCI/trunk/card/config_center_project.ini.init	(revision 2783)
+++ TOOLS/ConsoGENCI/trunk/card/config_center_project.ini.init	(revision 3083)
@@ -1,13 +1,13 @@
 [projet]
-name: project_name
+name = project_name
 
 [directories]
-ROOT_DIR: /path/to/ConsoGENCI
-DATA: %(ROOT_DIR)s/data
-PLOT: %(ROOT_DIR)s/plot
+ROOT_DIR = /path/to/ConsoGENCI
+DATA = %(ROOT_DIR)s/data
+PLOT = %(ROOT_DIR)s/plot
 DODS
 
 [dods]
-DIR: /path/to/dods/ConsoGENCI
-USER: user
-SERVER: host
+DIR = /path/to/dods/ConsoGENCI
+USER = user
+SERVER = host
Index: TOOLS/ConsoGENCI/trunk/launch_conso.sh
===================================================================
--- TOOLS/ConsoGENCI/trunk/launch_conso.sh	(revision 2783)
+++ TOOLS/ConsoGENCI/trunk/launch_conso.sh	(revision 3083)
@@ -13,7 +13,8 @@
 # ======================================================================
 
+date
+whereami=$( hostname -s )
+
 # set -vx
-
-whereami=$( hostname -s )
 
 # .. System dependant steps ..
@@ -51,5 +52,5 @@
 BIN_DIR=${ROOT_DIR}/bin
 
-declare -A proj_center=(["gencimp6"]="tgcc" ["rgzi"]="idris")
+declare -A proj_center=(["gencmip6"]="tgcc" ["rgzi"]="idris")
 
 for projet in "${!proj_center[@]}"
