Changeset 2420 for XIOS3

Timestamp:

10/13/22 15:45:43 (19 months ago)

Author:

jderouillat

Message:

Add an option (log_memory : set to false by default), to activate memory monitoring. Logs are now buffered.

Location:

XIOS3/trunk

Files:

• bld.cfg (modified) (1 diff)
• scripts/plotly_memory.js (modified) (1 diff)
• src/cxios.cpp (modified) (3 diffs)
• src/cxios.hpp (modified) (1 diff)
• src/mem_checker.cpp (modified) (5 diffs)
• src/mem_checker.hpp (modified) (2 diffs)
• src/server.cpp (modified) (1 diff)

XIOS3/trunk/bld.cfg

@@ -53 +53 @@
 bld::target xios_server.exe 
 bld::target generic_testcase.exe 
-bld::target test_ugrid.exe 
+#bld::target test_ugrid.exe 
 #bld::target test_regular.exe
 #bld::target test_xios2_cmip6.exe

XIOS3/trunk/scripts/plotly_memory.js

@@ -72 +72 @@
         title: "Memory consumption",
         xaxis: {
-            title: "Time (ms)",
+            title: "Time (s)",
         },
         yaxis: {

XIOS3/trunk/src/cxios.cpp

@@ -56 +56 @@
   bool CXios::checkSumRecv=false ;
   bool CXios::checkSumSend=false ;
+  bool CXios::logMemory=false ;
 
   CDaemonsManager*    CXios::daemonsManager_=nullptr ;
@@ -117 +118 @@
     checkSumSend = getin<bool>("checksum_send_fields", false);
     checkSumRecv = getin<bool>("checksum_recv_fields", false);
+    
+    logMemory = getin<bool>("log_memory", false);
 
     globalComm=MPI_COMM_WORLD ;
@@ -157 +160 @@
   void CXios::clientFinalize(void)
   {
-     CMemChecker::logMem("CXios::clientFinalize");
+     CMemChecker::logMem("CXios::clientFinalize", true);
 
      CClient::finalize() ;

XIOS3/trunk/src/cxios.hpp

@@ -68 +68 @@
      static bool checkSumRecv; //!< For debugging, compute a checksum of fields received by the model through the XIOS client
 
+     static bool logMemory; //!< Activate memory monitoring for all XIOS process
+    
      static const string defaultPoolId ;
      static const string defaultServerId ;

XIOS3/trunk/src/mem_checker.cpp

@@ -1 +1 @@
 #include "mem_checker.hpp"
+#include "cxios.hpp"
 #include "mpi.hpp"
 #include <string>
@@ -5 +6 @@
 #include <iostream>
 #include <sstream>
-#include <fstream>
 
 #include <fcntl.h>
@@ -20 +20 @@
   double CMemChecker::vsize_init_=0;
   double CMemChecker::time_init_=0;
-
+  std::ofstream CMemChecker::fout_;
+  int CMemChecker::flush_counter_=1;
+  
   CMemChecker::CMemChecker(const std::string& name) : name_(name) 
   { 
@@ -64 +66 @@
   void CMemChecker::logMem( std::string id, bool finalizeLog )
   {
-    // function get_xios_mem_data() {
-    //   return [
-    //        ...
-    //        [ "2000-01-01 01:00:10.XXX", "XIOS close context def", 1000],
-    //        [ "2000-01-01 01:00:11.XXX", "update timestep"       , 1000],
-    //        [ "2000-01-01 01:00:15.XXX", "send field"            , 2000],
-    //        ...
-    //   ];
-    // }
-    
-    std::ofstream fout;
+    if ( !CXios::logMemory ) return ;
+
     int rk = 0;
     MPI_Comm_rank( MPI_COMM_WORLD, &rk );
@@ -80 +73 @@
     double mem = getMemRSS();
     if (!mem) {
-      fout.open( logName );
-      fout << "time,event,memory" << std::endl;
-    }
-    else
-    {
-      fout.open( logName, std::ios_base::app );
+      fout_.open( logName );
+      fout_ << "time,event,memory" << std::endl;
     }
 
+    fout_.precision(4);
     // Time format : YYYY-MM-DD HH:MM:SS.XXX -> seconds * 1000.
-    fout << (MPI_Wtime()-time_init_)*1000. << "," << id << "," << mem/1000000. << std::endl;
+    fout_ << (MPI_Wtime()-time_init_) << "," << id << "," << mem/1000000. << std::endl;
 
-    fout.close();
+    if ((MPI_Wtime()-time_init_)>flush_counter_*600.)
+    {
+      fout_.flush();
+      flush_counter_++;
+    }
+    
+    if (finalizeLog)
+    {
+      fout_.close();
+    }
   }
 

XIOS3/trunk/src/mem_checker.hpp

@@ -4 +4 @@
 #include <string>
 #include <map>
+#include <fstream>
 
 namespace xios
@@ -39 +40 @@
       static double vsize_init_;
       static double time_init_;
+      static std::ofstream fout_;
+      static int flush_counter_;
   };
 }

XIOS3/trunk/src/server.cpp

@@ -434 +434 @@
       CXios::getMpiGarbageCollector().release() ; // release unfree MPI ressources
 
-      CMemChecker::logMem( "CServer::finalize" );
+      CMemChecker::logMem( "CServer::finalize", true );
       if (!is_MPI_Initialized)
       {