Changeset 606 for XIOS/branchs

Timestamp:

06/03/15 15:06:25 (9 years ago)

Author:

rlacroix

Message:

Support NetCDF4 compression.

Only available for non-parallel output so either if only one server is used or if the multiple file mode is enabled).

Location:

XIOS/branchs/xios-1.0/src

Files:

• config/field_attribute.conf (modified) (1 diff)
• config/file_attribute.conf (modified) (1 diff)
• output/nc4_data_output.cpp (modified) (1 diff)
• output/netCdfInterface.cpp (modified) (1 diff)
• output/netCdfInterface.hpp (modified) (1 diff)
• output/onetcdf4.cpp (modified) (3 diffs)
• output/onetcdf4.hpp (modified) (5 diffs)

XIOS/branchs/xios-1.0/src/config/field_attribute.conf

@@ -24 +24 @@
 DECLARE_ATTRIBUTE(double,    add_offset)
 DECLARE_ATTRIBUTE(double,    scale_factor)
+
+DECLARE_ATTRIBUTE(int,       compression_level)

XIOS/branchs/xios-1.0/src/config/file_attribute.conf

@@ -14 +14 @@
 DECLARE_ATTRIBUTE(StdString,      par_access)
 
-// DECLARE_ATTRIBUTE_DEF(bool, enabled, true)
+DECLARE_ATTRIBUTE(int,       compression_level)

XIOS/branchs/xios-1.0/src/output/nc4_data_output.cpp

@@ -736 +736 @@
               SuperClassWriter::setDefaultValue(fieldid, default_value);
            }
+
+           if (field->compression_level.isEmpty())
+             field->compression_level = field->file->compression_level.isEmpty() ? 0 : field->file->compression_level;
+           if (field->compression_level < 0 || field->compression_level > 9)
+             ERROR("void CNc4DataOutput::writeField_(CField* field)",
+                   "Invalid compression level, the value should range between 0 and 9.");
+           SuperClassWriter::setCompressionLevel(fieldid, field->compression_level);
 
            {  // Ecriture des coordonnées

XIOS/branchs/xios-1.0/src/output/netCdfInterface.cpp

@@ -525 +525 @@
 
 /*!
+This function sets the compression level to the specified variable
+\param [in] ncid Groud id (or file id)
+\param [in] varId Id of the variable
+\param [in] compressionLevel The compression level from 0 to 9 (0 disables the compression, 9 is the higher compression)
+\return Status code
+*/
+int CNetCdfInterface::defVarDeflate(int ncid, int varId, int compressionLevel)
+{
+  int status = nc_def_var_deflate(ncid, varId, false, (compressionLevel > 0), compressionLevel);
+  if (NC_NOERR != status)
+  {
+    StdString errormsg(nc_strerror(status));
+    StdStringStream sstr;
+
+    sstr << "Error in calling function " << "nc_def_var_deflate(ncid, varId, false, (compressionLevel > 0), compressionLevel)" << std::endl;
+    sstr << errormsg << std::endl;
+    sstr << "Unable to set the compression level of the variable with id: " << varId
+         << " and compression level: " << compressionLevel << std::endl;
+    StdString e = sstr.str();
+    throw CNetCdfException(e);
+  }
+
+  return status;
+}
+
+/*!
 This function makes a request to netcdf with a ncid, to set the fill parameters for a variable,
 given variable id and type of fill

XIOS/branchs/xios-1.0/src/output/netCdfInterface.hpp

@@ -90 +90 @@
     static int defVarChunking(int ncid, int varId, int storage, StdSize chunkSize[]);
 
+    //! Define variable compression level
+    static int defVarDeflate(int ncid, int varId, int compressionLevel);
+
     //! Define variable fill parameters
     static int defVarFill(int ncid, int varId, int noFill, void* fillValue);

XIOS/branchs/xios-1.0/src/output/onetcdf4.cpp

@@ -288 +288 @@
       //---------------------------------------------------------------
 
+      void CONetCDF4::setCompressionLevel(const StdString& varname, int compressionLevel)
+      {
+         int grpid = this->getCurrentGroup();
+         int varid = this->getVariable(varname);
+         CNetCdfInterface::defVarDeflate(grpid, varid, compressionLevel);
+      }
+
+      //---------------------------------------------------------------
+
       template <>
          void CONetCDF4::addAttribute
@@ -414 +423 @@
          {
             sstart.push_back(record);
-            scount.push_back(1); 
+            scount.push_back(1);
             if ((start == NULL) &&
                 (count == NULL)) i++;
@@ -421 +430 @@
 
          for (;it != end; it++)
-         { 
+         {
             if ((start != NULL) && (count != NULL))
             {

XIOS/branchs/xios-1.0/src/output/onetcdf4.hpp

@@ -46 +46 @@
             int addVariable(const StdString & name, nc_type type,
                             const std::vector<StdString> & dim);
-                            
+
       //----------------------------------------------------------------
          public :
-         
+
             template <class T>
                void setDefaultValue(const StdString & varname, const T * value = NULL);
-         
+
+            void setCompressionLevel(const StdString& varname, int compressionLevel);
+
             template <class T>  void addAttribute (const StdString & name, const T & value, const StdString * varname = NULL);
 
@@ -62 +64 @@
                               const std::vector<StdSize> * count = NULL);
 
-            void writeData(const CArray<int, 2>& data, const StdString & name);     
+            void writeData(const CArray<int, 2>& data, const StdString & name);
             void writeTimeAxisData(const CArray<double,1>& data, const StdString & name,
                                    bool collective, StdSize record, bool Isroot) ;
@@ -70 +72 @@
             /// Destructeur ///
             virtual ~CONetCDF4(void);
-            
+
       //----------------------------------------------------------------
-      
+
          protected :
 
@@ -93 +95 @@
 
       //----------------------------------------------------------------
-      
+
          private :
-         
+
             template <class T>
                void writeData_(int grpid, int varid,
@@ -118 +120 @@
 
       ///---------------------------------------------------------------
-           
+