Changeset 2751

Timestamp:

02/24/16 17:11:19 (8 years ago)

Author:

labetoulle

Message:

Homogenization of cpt files parsing, including multi-node projects on Curie

Location:

TOOLS/ConsoGENCI/trunk/bin

Files:

• init_alloc_tbl.py (modified) (4 diffs)
• init_conso_tbl.py (modified) (7 diffs)
• libconso.py (modified) (1 diff)
• libconso_cpt.py (added)
• libconso_db.py (added)

TOOLS/ConsoGENCI/trunk/bin/init_alloc_tbl.py

@@ -19 +19 @@
 
 
-########################################
+#######################################################################
 if __name__ == "__main__":
 
   project_list = [
     {
-      "name": "gencmip6",
+      "name": "gen0826",
       "centre": "tgcc",
       "machine": "curie",
-      "node": "thin",
-      "alloc": 5000000,
-      "start": "20150101",
-      "end": "20150630",
+      "node": "fat/large",
+      "alloc": 114000,
+      "start": "2015-01-01 00:00:00",
+      "end": "2015-12-31 23:59:59",
+    },
+    {
+      "name": "gen0826",
+      "centre": "tgcc",
+      "machine": "curie",
+      "node": "hybrid",
+      "alloc": 25000,
+      "start": "2015-01-01 00:00:00",
+      "end": "2015-12-31 23:59:59",
+    },
+    {
+      "name": "gen0826",
+      "centre": "tgcc",
+      "machine": "curie",
+      "node": "thin/standard",
+      "alloc": 114000,
+      "start": "2015-01-01 00:00:00",
+      "end": "2015-12-31 23:59:59",
     },
     {
@@ -36 +54 @@
       "centre": "tgcc",
       "machine": "curie",
-      "node": "thin",
+      "node": "thin/standard",
+      "alloc": 5000000,
+      "start": "2015-01-01 00:00:00",
+      "end": "2015-06-30 23:59:59",
+    },
+    {
+      "name": "gencmip6",
+      "centre": "tgcc",
+      "machine": "curie",
+      "node": "thin/standard",
       "alloc": 15000000,
-      "start": "20150701",
-      "end": "20151231",
+      "start": "2015-07-01 00:00:00",
+      "end": "2015-12-31 23:59:59",
+    },
+    {
+      "name": "gencmip6",
+      "centre": "tgcc",
+      "machine": "curie",
+      "node": "thin/standard",
+      "alloc": 25000000,
+      "start": "2016-01-01 00:00:00",
+      "end": "2016-06-30 23:59:59",
+    },
+    {
+      "name": "gencmip6",
+      "centre": "tgcc",
+      "machine": "curie",
+      "node": "thin/standard",
+      "alloc": 50000000,
+      "start": "2016-07-01 00:00:00",
+      "end": "2016-12-31 23:59:59",
     },
     {
@@ -47 +92 @@
       "node": "standard",
       "alloc": 2000000,
-      "start": "20150101",
-      "end": "20151231",
+      "start": "2015-01-01 00:00:00",
+      "end": "2015-12-31 23:59:59",
+    },
+    {
+      "name": "rgzi",
+      "centre": "idris",
+      "machine": "ada",
+      "node": "standard",
+      "alloc": 4000000,
+      "start": "2016-01-01 00:00:00",
+      "end": "2016-12-31 23:59:59",
     },
   ]
@@ -99 +153 @@
 
   cdb.commit_db(conn)
+
+  cdb.close_db(conn)

TOOLS/ConsoGENCI/trunk/bin/init_conso_tbl.py

@@ -1 +1 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-
-# ==================================================================== #
-# ssh readonly@prodiguer-test-db.ipsl.upmc.fr                          #
-# psql -U prodiguer_db_user prodiguer                                  #
-#                                                                      #
-# ssh readonly@prodiguer-test-db.ipsl.upmc.fr -L 5432:localhost:5432   #
-# ==================================================================== #
-
 
 # This must come first
@@ -18 +10 @@
 # import datetime as dt
 from argparse import ArgumentParser
+import pprint
 
 # Application library imports
-import libconsodb as cdb
+import libconso_db as cdb
+import libconso_cpt as ccpt
+
+pp = pprint.PrettyPrinter(indent=2)
 
 
-########################################
+#######################################################################
 def get_arguments():
   parser = ArgumentParser()
@@ -53 +49 @@
 
 
-########################################
+#######################################################################
 if __name__ == "__main__":
 
@@ -64 +60 @@
     print(args)
 
-  ROOT_DIR = "/home_local/slipsl/ConsoGENCMIP6/init_db"
+  # ... Files and directories ...
+  # -----------------------------
+  ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
   DATA_DIR = os.path.join(ROOT_DIR, "data")
-
   SUBMIT_DIR = os.getcwd()
 
-  # project = "rgzi"
-  # centre = "idris"
+  if args.verbose:
+    print("SUBMIT_DIR:", SUBMIT_DIR)
+    print("DATA_DIR:", DATA_DIR)
 
-  conso_per_login = {}
+  pattern = ccpt.cpt_pattern(args.center, args.project)
+  dirin = os.path.join(DATA_DIR, args.center, args.project)
+  filelist = ccpt.find_input_files(dirin, pattern, args.range)
 
-  # Connection info
+  # .. Connection to database ..
+  # ============================
+  if args.verbose:
+    print("Connection to database")
   db_host = "134.157.170.104"
   # db_port = "5432"
@@ -80 +83 @@
   db_user = "prodiguer_db_user"
   # db_pwd = "secret"
-
-  if args.center == "idris":
-    pattern = "compta_*.dat"
-  elif args.center == "tgcc":
-    pattern = "ccc_myproject.dat_*"
-  else:
-    print("Unknown center {}".format(args.center))
-    exit()
-
-  filelist = cdb.find_input_files(DATA_DIR, pattern, args.range)
-
-  # Build dictionary from files
-  # for filename in cdb.find_idris_files(DATA_DIR):
-  for filename in filelist:
-    print(filename)
-    # jour, heure, alloc, consos = \
-    date, alloc, consos = \
-          cdb.parse_input_cpt(filename, args.project, args.center)
-          # cdb.parse_idris_cpt(filename, args.project)
-
-    # date = dt.datetime.strptime("{} {}".format(jour, heure), "%d/%m/%Y %H:%M")
-
-    for item in consos:
-      login, conso = item
-      if login not in conso_per_login:
-        conso_per_login[login] = set()
-      conso_per_login[login].add((date, conso, ))
-
   conn, cursor = cdb.connect_db(db_host, db_name, db_user)
 
-  # Extract allocation id from table
+  # .. Extract allocation id from table ..
+  # ======================================
   table_name = "conso.tbl_allocation"
   request = (
-    "SELECT id "
+    # "SELECT id "
+    "SELECT * "
     "FROM " + table_name + " "
     "WHERE project = '" + args.project + "'"
     "  AND centre = '" + args.center + "'"
+    # "  AND start_date < '" + dt.datetime.strftime(date, "%Y-%m-%d") + "'"
+    # "  AND end_date > '" + dt.datetime.strftime(date, "%Y-%m-%d") + "'"
+    "ORDER BY start_date"
     ";"
   )
 
   cdb.select_db(cursor, request)
-  if cursor.rowcount != 1:
-    print(cursor.fetchall())
-    exit()
+  # print(cursor.rowcount)
+  # print(cursor.fetchall())
 
-  (allocation_id, ) = cursor.fetchone()
+  allocs = []
+  for row in cursor:
+    allocs.append(ccpt.AllocRow(row))
 
-  for login, conso_list in conso_per_login.iteritems():
+  # .. Build dictionary from files ..
+  # =================================
+  conso_per_alloc = {}
+  for filename in filelist:
+    date, blocs = ccpt.parse_input_cpt(
+      filename, args.project, args.center, mode_conso=True
+    )
 
-    # test = "'"+login+"'" if login != "total" else "NULL"
-    # print(test)
+    for bloc in blocs:
+      if not bloc.alloc_id:
+        bloc.alloc_id = ccpt.get_project_id(bloc, allocs)
+        if not bloc.alloc_id:
+          print("no alloc id found, skip bloc")
+          print(bloc.machine, bloc.node, bloc.cpt_date)
+          pp.pprint(allocs)
+          continue
+      if bloc.alloc_id not in conso_per_alloc:
+        conso_per_alloc[bloc.alloc_id] = set()
+      for conso in bloc.consos:
+        conso_per_alloc[bloc.alloc_id].add(conso)
 
+  # .. Insert data in table, one alloc_id at a time ..
+  # ==================================================
+  for alloc_id, consos in conso_per_alloc.iteritems():
+    # ... Create request sub string ...
+    # ---------------------------------
     lines_req = [
       (
@@ -140 +142 @@
         "{create})"
       ) .format(
-        alloc=allocation_id,
-        date=date,
-        total_hrs=conso if not math.isnan(conso) else "'NaN'",
-        login="'"+login+"'" if login != "total" else "NULL",
+        alloc=alloc_id,
+        date=item.date,
+        total_hrs=item.conso if not math.isnan(item.conso) else "'NaN'",
+        login="'"+item.login+"'" if item.login != "total" else "NULL",
         create="CURRENT_TIMESTAMP",
-      ) for (date, conso) in conso_list
+      ) for item in consos
     ]
 
+    # print(lines_req)
+
+    # exit()
+
+    # ... Create full request ...
+    # ---------------------------
     table_name = "conso.tbl_consumption"
     request = (
@@ -160 +168 @@
     )
     request = request + ", ".join(lines_req)
-    print(request)
 
+    # ... Execute request ...
+    # -----------------------
+    if args.verbose:
+      print("Execute request for alloc_id = {}".format(alloc_id))
     cdb.insert_db(cursor, request)
 
+    # ... Commit inserts ...
+    # ----------------------
     if not args.dryrun:
+      if args.verbose:
+        print("Commit inserts")
       cdb.commit_db(conn)
+
+  # .. Close connection to database ..
+  # ==================================
+  if args.verbose:
+    print("Close connection")
+  cdb.close_db(conn)

TOOLS/ConsoGENCI/trunk/bin/libconso.py

@@ -13 +13 @@
 import subprocess
 import datetime as dt
-import numpy as np
+# import numpy as np
 import ConfigParser as cp