Changeset 1328 for XIOS

XIOS/dev/branch_openmp/bld.cfg

-                      r1287
+                      r1328
 #bld::target test_new_features.exe
 #bld::target test_unstruct_complete.exe
+bld::target test_omp.exe
+bld::target test_complete_omp.exe
+bld::target test_remap_omp.exe
+bld::target test_unstruct_omp.exe
+#bld::target test_omp.exe
+#bld::target test_complete_omp.exe
+bld::target test_remap.exe
+bld::target test_remap_ref.exe
+#bld::target test_unstruct_omp.exe
 #bld::target test_netcdf_omp.exe
 #bld::target test_client.exe
 #bld::target test_complete.exe
+bld::target test_client.exe
+bld::target test_complete.exe
 #bld::target test_remap.exe
 #bld::target test_unstruct.exe

XIOS/dev/branch_openmp/extern/remap/src/clipper.cpp

-                      r1205
+                      r1328
+{
   //The equation of a line in general form (Ax + By + C = 0)
   //given 2 points (x,y) & (x,y) is ...
   //(y - y)x + (x - x)y + (y - y)x - (x - x)y = 0
   //A = (y - y); B = (x - x); C = (y - y)x - (x - x)y
   //perpendicular distance of point (x,y) = (Ax + By + C)/Sqrt(A + B)
+  //given 2 points (x¹,y¹) & (x²,y²) is ...
+  //(y¹ - y²)x + (x² - x¹)y + (y² - y¹)x¹ - (x² - x¹)y¹ = 0
+  //A = (y¹ - y²); B = (x² - x¹); C = (y² - y¹)x¹ - (x² - x¹)y¹
+  //perpendicular distance of point (x³,y³) = (Ax³ + By³ + C)/Sqrt(A² + B²)
   //see http://en.wikipedia.org/wiki/Perpendicular_distance
   double A = double(ln1.Y - ln2.Y);

XIOS/dev/branch_openmp/extern/remap/src/cputime.cpp

r694	r1328
1	1	#include "mpi.hpp"
	2	using namespace ep_lib;
2	3
3	4	namespace sphereRemap {

XIOS/dev/branch_openmp/extern/remap/src/elt.hpp

-                      r1172
+                      r1328
 struct Elt : Polyg
+{
+  Elt() {}
+  Elt(const double *bounds_lon, const double *bounds_lat, int max_num_vert)
+  {
+    int k = 0;
+    vertex[k++] = xyz(bounds_lon[0], bounds_lat[0]);
+    for (int i = 1; i < max_num_vert; i++)
+    {
+      vertex[k] = xyz(bounds_lon[i], bounds_lat[i]);
+      /* netCDF convention: if first vertex repeats element is finished (at least three vertices == triagle) */
+      if (k >= 3 && squaredist(vertex[k], vertex[0]) < EPS*EPS)
+        break;
+      /* eliminate zero edges: move to next vertex only if it is different */
+      if (squaredist(vertex[k], vertex[k-1]) > EPS*EPS)
+        k++;
+      //else cout << "Removed edge " << k << " due to zero length (coinciding endpoints)." << endl ;
+    }
+    n = k;
+    x = barycentre(vertex, n);
+  }
+        Elt() {}
+        Elt(const double *bounds_lon, const double *bounds_lat, int max_num_vert)
+        {
+                int k = 0;
+                vertex[k++] = xyz(bounds_lon[0], bounds_lat[0]);
+                for (int i = 1; i < max_num_vert; i++)
+                {
+                        vertex[k] = xyz(bounds_lon[i], bounds_lat[i]);
+                        /* netCDF convention: if first vertex repeats element is finished (at least three vertices == triagle) */
+                        if (k >= 3 && squaredist(vertex[k], vertex[0]) < EPS*EPS)
+                                break;
+                        /* eliminate zero edges: move to next vertex only if it is different */
+                        if (squaredist(vertex[k], vertex[k-1]) > EPS*EPS)
+                                k++;
+                        else
+                                /* cout << "Removed edge " << k << " due to zero length (coinciding endpoints)." << endl */ ;
+                }
+                n = k;
+                x = barycentre(vertex, n);
+        }
         Elt& operator=(const Elt& rhs)
 …
+        }
   void delete_intersections()
+  {
     for (list<Polyg*>::iterator it = this->is.begin(); it != this->is.end(); it++)
+    {
       Polyg* poly = *it;
       delete poly;
+    }
+  }
+        void delete_intersections()
+        {
+                for (list<Polyg*>::iterator it = this->is.begin(); it != this->is.end(); it++)
+                {
+                        Polyg* poly = *it;
+                        delete poly;
+                }
+        }
   void insert_vertex(int i, const Coord& v)

XIOS/dev/branch_openmp/extern/remap/src/gridRemap.cpp

-                      r1155
+                      r1328
 CRemapGrid srcGrid;
-#pragma omp threadprivate(srcGrid)
 CRemapGrid tgtGrid;
-#pragma omp threadprivate(tgtGrid)
+}

XIOS/dev/branch_openmp/extern/remap/src/gridRemap.hpp

-                      r1220
+                      r1328
 Coord readPole(std::istream&);
+extern CRemapGrid srcGrid;
+extern CRemapGrid tgtGrid;
+}

XIOS/dev/branch_openmp/extern/remap/src/intersect.cpp

-                      r1220
+                      r1328
 namespace sphereRemap {
-extern CRemapGrid srcGrid;
-#pragma omp threadprivate(srcGrid)
-extern CRemapGrid tgtGrid;
-#pragma omp threadprivate(tgtGrid)
 using namespace std;
 …
 int neighbour_idx(const Elt& a, const Elt& b)
+{
   for (int i = 0; i < a.n; i++)
+  {
     for (int j = 0; j < b.n; j++)
+    {
       assert(squaredist(a.vertex[ i       ], b.vertex[ j       ]) > EPS*EPS ||
              squaredist(a.vertex[(i+1)%a.n], b.vertex[(j+1)%b.n]) > EPS*EPS);
       if (   squaredist(a.vertex[ i       ], b.vertex[ j           ]) < 1e-13*1e-13 &&
              squaredist(a.vertex[(i+1)%a.n], b.vertex[(j+b.n-1)%b.n]) < 1e-13*1e-13)
+      {
         return i;
+      }
+    }
+  }
   return NOT_FOUND;
+        for (int i = 0; i < a.n; i++)
+        {
+                for (int j = 0; j < b.n; j++)
+                {
+                        assert(squaredist(a.vertex[ i       ], b.vertex[ j       ]) > EPS*EPS ||
+                               squaredist(a.vertex[(i+1)%a.n], b.vertex[(j+1)%b.n]) > EPS*EPS);
+                        if (   squaredist(a.vertex[ i       ], b.vertex[ j           ]) < 1e-13*1e-13 &&
+                               squaredist(a.vertex[(i+1)%a.n], b.vertex[(j+b.n-1)%b.n]) < 1e-13*1e-13)
+                        {
+                                return i;
+                        }
+                }
+        }
+        return NOT_FOUND;
+}
 …
 bool isNeighbour(Elt& a, const Elt& b)
+{
   // return neighbour_idx(a, b) != NOT_FOUND;
+        // return neighbour_idx(a, b) != NOT_FOUND;
   return insertNeighbour(a,b,false) ;
+}
 …
 void intersect(Elt *a, Elt *b)
+{
   int na = a->n; /* vertices of a */
   int nb = b->n; /* vertices of b */
   Coord *c   = new Coord[na+nb];
   Coord *c2  = new Coord[na+nb];
   Coord *xc  = new Coord[na+nb];
   Coord *xc2 = new Coord[na+nb];
   Coord gc, gc2;
   double *d = new double[na+nb];
   double *d2 = new double[na+nb];
   double are, are2;
   Ipt ipt[NMAX*NMAX];
   Ipt ipt2[NMAX*NMAX];
   ptsec(a, b, ipt);
   /* make ipt2 transpose of ipt */
   for (int ii = 0; ii < na; ii++)
     for (int jj = 0; jj < nb; jj++)
       ipt2[jj*na+ii] = ipt[ii*nb+jj];
   list<Sgm> iscot;
   recense(a, b, ipt, iscot, 0);
   recense(b, a, ipt2, iscot, 1);
   int nseg = iscot.size();
   int nc = 0;
   int nc2 = 0;
   while (iscot.size() && nc < 2)
     nc = assemble(iscot, c, d, xc);
   while (iscot.size() && nc2 < 2)
     nc2 = assemble(iscot, c2, d2, xc2);
 //  assert(nseg == nc + nc2 || nseg == 1); // unused segment
+        int na = a->n; /* vertices of a */
+        int nb = b->n; /* vertices of b */
+        Coord *c   = new Coord[na+nb];
+        Coord *c2  = new Coord[na+nb];
+        Coord *xc  = new Coord[na+nb];
+        Coord *xc2 = new Coord[na+nb];
+        Coord gc, gc2;
+        double *d = new double[na+nb];
+        double *d2 = new double[na+nb];
+        double are, are2;
+        Ipt ipt[NMAX*NMAX];
+        Ipt ipt2[NMAX*NMAX];
+        ptsec(a, b, ipt);
+        /* make ipt2 transpose of ipt */
+        for (int ii = 0; ii < na; ii++)
+                for (int jj = 0; jj < nb; jj++)
+                        ipt2[jj*na+ii] = ipt[ii*nb+jj];
+        list<Sgm> iscot;
+        recense(a, b, ipt, iscot, 0);
+        recense(b, a, ipt2, iscot, 1);
+        int nseg = iscot.size();
+        int nc = 0;
+        int nc2 = 0;
+        while (iscot.size() && nc < 2)
+                nc = assemble(iscot, c, d, xc);
+        while (iscot.size() && nc2 < 2)
+                nc2 = assemble(iscot, c2, d2, xc2);
+//      assert(nseg == nc + nc2 || nseg == 1); // unused segment
         if (!(nseg == nc + nc2 || nseg == 1))
 …
 //    intersect_ym(a,b) ;
   if (nc == 1) nc = 0;
   if (nc2 == 1) nc2 = 0;
   gc = barycentre(xc, nc);
   gc2 = barycentre(xc2, nc2);
   orient(nc, xc, c, d, gc);
   Coord pole = srcGrid.pole;
   if (pole == ORIGIN) pole = tgtGrid.pole;
   const double MINBASE = 1e-11;
   if (nc == 2) /* nc is the number of vertices of super mesh element */
+  {
     double base = arcdist(xc[0], xc[1]);
+        if (nc == 1) nc = 0;
+        if (nc2 == 1) nc2 = 0;
+        gc = barycentre(xc, nc);
+        gc2 = barycentre(xc2, nc2);
+        orient(nc, xc, c, d, gc);
+        Coord pole = srcGrid.pole;
+        if (pole == ORIGIN) pole = tgtGrid.pole;
+        const double MINBASE = 1e-11;
+        if (nc == 2) /* nc is the number of vertices of super mesh element */
+        {
+                double base = arcdist(xc[0], xc[1]);
 cerr << "DID ARRIVE " << base << xc[0] << xc[1] << endl;
     gc = midpoint(gc, midpointSC(xc[0], xc[1]));
     /* intersection area `are` must be zero here unless we have one great and one small circle */
     are = alun(base, fabs(scalarprod(xc[0], pole)));
+  }
   else
+  {
     are = airbar(nc, xc, c, d, pole, gc);
+  }
   if (nc2 == 2)
+  {
     double base = arcdist(xc2[0], xc2[1]);
+                gc = midpoint(gc, midpointSC(xc[0], xc[1]));
+                /* intersection area `are` must be zero here unless we have one great and one small circle */
+                are = alun(base, fabs(scalarprod(xc[0], pole)));
+        }
+        else
+        {
+                are = airbar(nc, xc, c, d, pole, gc);
+        }
+        if (nc2 == 2)
+        {
+                double base = arcdist(xc2[0], xc2[1]);
 cerr << "DID ARRIVE " << base << xc2[0] << xc2[1] << endl;
     assert(base > MINBASE);
     gc2 = midpoint(gc2, midpointSC(xc2[0], xc2[1]));
     are2 = alun(base, fabs(scalarprod(xc2[0], pole))); // 0
+  }
   else
+  {
     are2 = airbar(nc2, xc2, c2, d2, pole, gc2);
+  }
+                assert(base > MINBASE);
+                gc2 = midpoint(gc2, midpointSC(xc2[0], xc2[1]));
+                are2 = alun(base, fabs(scalarprod(xc2[0], pole))); // 0
+        }
+        else
+        {
+                are2 = airbar(nc2, xc2, c2, d2, pole, gc2);
+        }
 //  double ym_area=intersect_ym(a,b) ;
   if (nc > 1)
+  {
     /* create one super mesh polygon that src and dest point to */
     Polyg *is = new Polyg;
     is->x = gc;
     is->area = are;
     is->id = b->id;
     is->src_id = b->src_id;
     is->n = nc;
     (a->is).push_back(is);
     (b->is).push_back(is);
+        {
+                /* create one super mesh polygon that src and dest point to */
+                Polyg *is = new Polyg;
+                is->x = gc;
+                is->area = are;
+                is->id = b->id;
+                is->src_id = b->src_id;
+                is->n = nc;
+                (a->is).push_back(is);
+                (b->is).push_back(is);
 /*
     if (  2*fabs(are-ym_area)/(are+ym_area) > 1.1 && ym_area>1e-8)
+          if (  2*fabs(are-ym_area)/(are+ym_area) > 1.1 && ym_area>1e-8)
+    {
       cout<<"Big area difference : "<<are<<"  "<<ym_area<<endl ;
 …
+    }
 */
 //    cout<<"intersection : "<<are<<" "<< ym_area<<"  diff : "<<fabs(are-ym_area)<<"  ratio : "<<fabs(are-ym_area)/(0.5*(are+ym_area))<<endl ;
+  }
   if (nc2 > 1)
+  {
     Polyg *is = new Polyg;
     is->x = gc2;
     is->area = are2;
     is->id = b->id; /* intersection holds id of corresponding source element (see Elt class definition for details about id) */
     is->src_id = b->src_id;
     is->n = nc2;
     (a->is).push_back(is);
     (b->is).push_back(is);
+//              cout<<"intersection : "<<are<<" "<< ym_area<<"  diff : "<<fabs(are-ym_area)<<"  ratio : "<<fabs(are-ym_area)/(0.5*(are+ym_area))<<endl ;
+        }
+        if (nc2 > 1)
+        {
+                Polyg *is = new Polyg;
+                is->x = gc2;
+                is->area = are2;
+                is->id = b->id; /* intersection holds id of corresponding source element (see Elt class definition for details about id) */
+                is->src_id = b->src_id;
+                is->n = nc2;
+                (a->is).push_back(is);
+                (b->is).push_back(is);
 /*
     if (  2*fabs(are-ym_area)/(are+ym_area) > 1.1 && ym_area>1e-8 )
+    if (        2*fabs(are-ym_area)/(are+ym_area) > 1.1 && ym_area>1e-8 )
+    {
       cout<<"Big area difference : "<<are<<"  "<<ym_area<<endl ;
 …
+    }
 */
 //    cout<<"intersection : "<<are2<<" "<< ym_area<<"  diff : "<<fabs(are-ym_area)<<"  ratio : "<<fabs(are-ym_area)/(0.5*(are+ym_area))<<endl ;
+  }
+//              cout<<"intersection : "<<are2<<" "<< ym_area<<"  diff : "<<fabs(are-ym_area)<<"  ratio : "<<fabs(are-ym_area)/(0.5*(are+ym_area))<<endl ;
+        }
 /*
   if (nc<=1 && nc2<=1)
 …
+  }
 */
   delete [] c;
   delete [] c2;
   delete [] xc;
   delete [] xc2;
   delete [] d;
   delete [] d2;
+}
+}
+        delete [] c;
+        delete [] c2;
+        delete [] xc;
+        delete [] xc2;
+        delete [] d;
+        delete [] d2;
+}
+}

XIOS/dev/branch_openmp/extern/remap/src/intersection_ym.cpp

-                      r1220
+                      r1328
 namespace sphereRemap {
-extern CRemapGrid srcGrid;
-#pragma omp threadprivate(srcGrid)
-extern CRemapGrid tgtGrid;
-#pragma omp threadprivate(tgtGrid)
 using namespace std;

XIOS/dev/branch_openmp/extern/remap/src/libmapper.cpp

-                      r1205
+                      r1328
 #include "mapper.hpp"
 #include "cputime.hpp" // cputime
+#include <stdio.h>
+using namespace ep_lib;
 using namespace sphereRemap ;
 …
    and deallocated during the second step (computing the weights) */
 Mapper *mapper;
+#pragma omp threadprivate(mapper)
 /** xxx_bounds_yyy is of length n_vert_per_cell_xxx*n_cell_xxx
 …
                      int order, int* n_weights)
+{
   assert(src_bounds_lon);
   assert(src_bounds_lat);
   assert(n_vert_per_cell_src >= 3);
   assert(n_cell_src >= 4);
   assert(dst_bounds_lon);
   assert(dst_bounds_lat);
   assert(n_vert_per_cell_dst >= 3);
   assert(n_cell_dst >= 4);
   assert(1 <= order && order <= 2);
+        assert(src_bounds_lon);
+        assert(src_bounds_lat);
+        assert(n_vert_per_cell_src >= 3);
+        assert(n_cell_src >= 4);
+        assert(dst_bounds_lon);
+        assert(dst_bounds_lat);
+        assert(n_vert_per_cell_dst >= 3);
+        assert(n_cell_dst >= 4);
+        assert(1 <= order && order <= 2);
   mapper = new Mapper(MPI_COMM_WORLD);
 …
         double tic = cputime();
         mapper = new Mapper(MPI_COMM_WORLD);
         mapper->setVerbosity(PROGRESS) ;
+  mapper->setVerbosity(PROGRESS) ;
         mapper->buildSSTree(src_msh, dst_msh);
         double tac = cputime();
 …
         char **argv = NULL;
         MPI_Init(&argc, &argv);*/
+        //MPI_Init(NULL, NULL);
+        int provided;
+        MPI_Init_thread(NULL, NULL, 3, &provided);
+        assert(provided >= 3);
+        MPI_Init(NULL, NULL);
+}

XIOS/dev/branch_openmp/extern/remap/src/mapper.cpp

-                      r1220
+                      r1328
 #include "mapper.hpp"
+using namespace ep_lib;
 namespace sphereRemap {
-extern CRemapGrid srcGrid;
-#pragma omp threadprivate(srcGrid)
-extern CRemapGrid tgtGrid;
-#pragma omp threadprivate(tgtGrid)
 /* A subdivition of an array into N sub-arays
 …
 void Mapper::setTargetMesh(const double* boundsLon, const double* boundsLat, int nVertex, int nbCells, const double* pole, const long int* globalId)
+{
     tgtGrid.pole = Coord(pole[0], pole[1], pole[2]);
     int mpiRank, mpiSize;
     MPI_Comm_rank(communicator, &mpiRank);
     MPI_Comm_size(communicator, &mpiSize);
     targetElements.reserve(nbCells);
     targetMesh.reserve(nbCells);
     targetGlobalId.resize(nbCells) ;
     if (globalId==NULL)
+    {
         long int offset ;
         long int nb=nbCells ;
         MPI_Scan(&nb,&offset,1,MPI_LONG,MPI_SUM,communicator) ;
         offset=offset-nb ;
         for(int i=0;i<nbCells;i++) targetGlobalId[i]=offset+i ;
+    }
     else targetGlobalId.assign(globalId,globalId+nbCells);
     for (int i = 0; i < nbCells; i++)
+    {
         int offs = i*nVertex;
         Elt elt(boundsLon + offs, boundsLat + offs, nVertex);
         targetElements.push_back(elt);
         targetMesh.push_back(Node(elt.x, cptRadius(elt), &sourceElements.back()));
         cptEltGeom(targetElements[i], Coord(pole[0], pole[1], pole[2]));
+    }
+  tgtGrid.pole = Coord(pole[0], pole[1], pole[2]);
+  int mpiRank, mpiSize;
+  MPI_Comm_rank(communicator, &mpiRank);
+  MPI_Comm_size(communicator, &mpiSize);
+  targetElements.reserve(nbCells);
+  targetMesh.reserve(nbCells);
+  targetGlobalId.resize(nbCells) ;
+  if (globalId==NULL)
+  {
+    long int offset ;
+    long int nb=nbCells ;
+    MPI_Scan(&nb,&offset,1,MPI_LONG,MPI_SUM,communicator) ;
+    offset=offset-nb ;
+    for(int i=0;i<nbCells;i++) targetGlobalId[i]=offset+i ;
+  }
+  else targetGlobalId.assign(globalId,globalId+nbCells);
+  for (int i = 0; i < nbCells; i++)
+  {
+    int offs = i*nVertex;
+    Elt elt(boundsLon + offs, boundsLat + offs, nVertex);
+    targetElements.push_back(elt);
+    targetMesh.push_back(Node(elt.x, cptRadius(elt), &sourceElements.back()));
+    cptEltGeom(targetElements[i], Coord(pole[0], pole[1], pole[2]));
+  }
 …
 void Mapper::setSourceValue(const double* val)
+{
     int size=sourceElements.size() ;
     for(int i=0;i<size;++i) sourceElements[i].val=val[i] ;
+  int size=sourceElements.size() ;
+  for(int i=0;i<size;++i) sourceElements[i].val=val[i] ;
+}
 void Mapper::getTargetValue(double* val)
+{
     int size=targetElements.size() ;
     for(int i=0;i<size;++i) val[i]=targetElements[i].val ;
+  int size=targetElements.size() ;
+  for(int i=0;i<size;++i) val[i]=targetElements[i].val ;
+}
 vector<double> Mapper::computeWeights(int interpOrder, bool renormalize, bool quantity)
+{
+    vector<double> timings;
+    int mpiSize, mpiRank;
+    MPI_Comm_size(communicator, &mpiSize);
+    MPI_Comm_rank(communicator, &mpiRank);
+    this->buildSSTree(sourceMesh, targetMesh);
+    if (mpiRank == 0 && verbose) cout << "Computing intersections ..." << endl;
+    double tic = cputime();
+    computeIntersection(&targetElements[0], targetElements.size());
+    timings.push_back(cputime() - tic);
+    tic = cputime();
+    if (interpOrder == 2) {
+        if (mpiRank == 0 && verbose) cout << "Computing grads ..." << endl;
+        buildMeshTopology();
+        computeGrads();
+    }
+    timings.push_back(cputime() - tic);
+    /* Prepare computation of weights */
+    /* compute number of intersections which for the first order case
+       corresponds to the number of edges in the remap matrix */
+    int nIntersections = 0;
+    for (int j = 0; j < targetElements.size(); j++)
+    {
+        Elt &elt = targetElements[j];
+        for (list<Polyg*>::iterator it = elt.is.begin(); it != elt.is.end(); it++)
+            nIntersections++;
+    }
+    /* overallocate for NMAX neighbours for each elements */
+    remapMatrix = new double[nIntersections*NMAX];
+    srcAddress = new int[nIntersections*NMAX];
+    srcRank = new int[nIntersections*NMAX];
+    dstAddress = new int[nIntersections*NMAX];
+    sourceWeightId =new long[nIntersections*NMAX];
+    targetWeightId =new long[nIntersections*NMAX];
+    if (mpiRank == 0 && verbose) cout << "Remapping..." << endl;
+    tic = cputime();
+    nWeights = remap(&targetElements[0], targetElements.size(), interpOrder, renormalize, quantity);
+    timings.push_back(cputime() - tic);
+    for (int i = 0; i < targetElements.size(); i++) targetElements[i].delete_intersections();
+    return timings;
+  vector<double> timings;
+  int mpiSize, mpiRank;
+  MPI_Comm_size(communicator, &mpiSize);
+  MPI_Comm_rank(communicator, &mpiRank);
+  this->buildSSTree(sourceMesh, targetMesh);
+  if (mpiRank == 0 && verbose) cout << "Computing intersections ..." << endl;
+  double tic = cputime();
+  computeIntersection(&targetElements[0], targetElements.size());
+  timings.push_back(cputime() - tic);
+        tic = cputime();
+        if (interpOrder == 2) {
+                if (mpiRank == 0 && verbose) cout << "Computing grads ..." << endl;
+                buildMeshTopology();
+                computeGrads();
+        }
+        timings.push_back(cputime() - tic);
+        /* Prepare computation of weights */
+        /* compute number of intersections which for the first order case
+           corresponds to the number of edges in the remap matrix */
+        int nIntersections = 0;
+        for (int j = 0; j < targetElements.size(); j++)
+        {
+                Elt &elt = targetElements[j];
+                for (list<Polyg*>::iterator it = elt.is.begin(); it != elt.is.end(); it++)
+                        nIntersections++;
+        }
+        /* overallocate for NMAX neighbours for each elements */
+        remapMatrix = new double[nIntersections*NMAX];
+        srcAddress = new int[nIntersections*NMAX];
+        srcRank = new int[nIntersections*NMAX];
+        dstAddress = new int[nIntersections*NMAX];
+  sourceWeightId =new long[nIntersections*NMAX];
+  targetWeightId =new long[nIntersections*NMAX];
+        if (mpiRank == 0 && verbose) cout << "Remapping..." << endl;
+        tic = cputime();
+        nWeights = remap(&targetElements[0], targetElements.size(), interpOrder, renormalize, quantity);
+        timings.push_back(cputime() - tic);
+  for (int i = 0; i < targetElements.size(); i++) targetElements[i].delete_intersections();
+        return timings;
+}
 /**
   @param elements are cells of the target grid that are distributed over CPUs
   indepentently of the distribution of the SS-tree.
   @param nbElements is the size of the elements array.
   @param order is the order of interpolaton (must be 1 or 2).
   */
+   @param elements are cells of the target grid that are distributed over CPUs
+          indepentently of the distribution of the SS-tree.
+   @param nbElements is the size of the elements array.
+   @param order is the order of interpolaton (must be 1 or 2).
+*/
 int Mapper::remap(Elt *elements, int nbElements, int order, bool renormalize, bool quantity)
+{
+    int mpiSize, mpiRank;
+    MPI_Comm_size(communicator, &mpiSize);
+    MPI_Comm_rank(communicator, &mpiRank);
+    /* create list of intersections (super mesh elements) for each rank */
+    multimap<int, Polyg *> *elementList = new multimap<int, Polyg *>[mpiSize];
+    for (int j = 0; j < nbElements; j++)
+    {
+        Elt& e = elements[j];
+        for (list<Polyg *>::iterator it = e.is.begin(); it != e.is.end(); it++)
+            elementList[(*it)->id.rank].insert(pair<int, Polyg *>((*it)->id.ind, *it));
+    }
+    int *nbSendElement = new int[mpiSize];
+    int **sendElement = new int*[mpiSize]; /* indices of elements required from other rank */
+    double **recvValue = new double*[mpiSize];
+    double **recvArea = new double*[mpiSize];
+    Coord **recvGrad = new Coord*[mpiSize];
+    GloId **recvNeighIds = new GloId*[mpiSize]; /* ids of the of the source neighbours which also contribute through gradient */
+    for (int rank = 0; rank < mpiSize; rank++)
+    {
+        /* get size for allocation */
+        int last = -1; /* compares unequal to any index */
+        int index = -1; /* increased to starting index 0 in first iteration */
+        for (multimap<int, Polyg *>::iterator it = elementList[rank].begin(); it != elementList[rank].end(); ++it)
+        {
+            if (last != it->first)
+                index++;
+            (it->second)->id.ind = index;
+            last = it->first;
+        }
+        nbSendElement[rank] = index + 1;
+        /* if size is non-zero allocate and collect indices of elements on other ranks that we intersect */
+        if (nbSendElement[rank] > 0)
+        {
+            sendElement[rank] = new int[nbSendElement[rank]];
+            recvValue[rank]   = new double[nbSendElement[rank]];
+            recvArea[rank]    = new double[nbSendElement[rank]];
+            if (order == 2)
+            {
+                recvNeighIds[rank] = new GloId[nbSendElement[rank]*(NMAX+1)];
+                recvGrad[rank]    = new Coord[nbSendElement[rank]*(NMAX+1)];
+            }
+            else
+                recvNeighIds[rank] = new GloId[nbSendElement[rank]];
+            last = -1;
+            index = -1;
+            for (multimap<int, Polyg *>::iterator it = elementList[rank].begin(); it != elementList[rank].end(); ++it)
+            {
+                if (last != it->first)
+                    index++;
+                sendElement[rank][index] = it->first;
+                last = it->first;
+            }
+        }
+    }
+    /* communicate sizes of source elements to be sent (index lists and later values and gradients) */
+    int *nbRecvElement = new int[mpiSize];
+    MPI_Alltoall(nbSendElement, 1, MPI_INT, nbRecvElement, 1, MPI_INT, communicator);
+    /* communicate indices of source elements on other ranks whoes value and gradient we need (since intersection) */
+    int nbSendRequest = 0;
+    int nbRecvRequest = 0;
+    int **recvElement = new int*[mpiSize];
+    double **sendValue = new double*[mpiSize];
+    double **sendArea = new double*[mpiSize];
+    Coord **sendGrad = new Coord*[mpiSize];
+    GloId **sendNeighIds = new GloId*[mpiSize];
+    MPI_Request *sendRequest = new MPI_Request[4*mpiSize];
+    MPI_Request *recvRequest = new MPI_Request[4*mpiSize];
+    for (int rank = 0; rank < mpiSize; rank++)
+    {
+        if (nbSendElement[rank] > 0)
+        {
+            MPI_Issend(sendElement[rank], nbSendElement[rank], MPI_INT, rank, 0, communicator, &sendRequest[nbSendRequest]);
+            nbSendRequest++;
+        }
+        if (nbRecvElement[rank] > 0)
+        {
+            recvElement[rank] = new int[nbRecvElement[rank]];
+            sendValue[rank]   = new double[nbRecvElement[rank]];
+            sendArea[rank]   = new double[nbRecvElement[rank]];
+            if (order == 2)
+            {
+                sendNeighIds[rank] = new GloId[nbRecvElement[rank]*(NMAX+1)];
+                sendGrad[rank]    = new Coord[nbRecvElement[rank]*(NMAX+1)];
+            }
+            else
+            {
+                sendNeighIds[rank] = new GloId[nbRecvElement[rank]];
+            }
+            MPI_Irecv(recvElement[rank], nbRecvElement[rank], MPI_INT, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+            nbRecvRequest++;
+        }
+    }
+    MPI_Status *status = new MPI_Status[4*mpiSize];
+    MPI_Waitall(nbSendRequest, sendRequest, status);
+    MPI_Waitall(nbRecvRequest, recvRequest, status);
+    /* for all indices that have been received from requesting ranks: pack values and gradients, then send */
+    nbSendRequest = 0;
+    nbRecvRequest = 0;
+    for (int rank = 0; rank < mpiSize; rank++)
+    {
+        if (nbRecvElement[rank] > 0)
+        {
+            int jj = 0; // jj == j if no weight writing
+            for (int j = 0; j < nbRecvElement[rank]; j++)
+            {
+                sendValue[rank][j] = sstree.localElements[recvElement[rank][j]].val;
+                sendArea[rank][j] = sstree.localElements[recvElement[rank][j]].area;
+                if (order == 2)
+                {
+                    sendGrad[rank][jj] = sstree.localElements[recvElement[rank][j]].grad;
+                    sendNeighIds[rank][jj] = sstree.localElements[recvElement[rank][j]].src_id;
+                    jj++;
+                    for (int i = 0; i < NMAX; i++)
+                    {
+                        sendGrad[rank][jj] = sstree.localElements[recvElement[rank][j]].gradNeigh[i];
+                        sendNeighIds[rank][jj] = sstree.localElements[recvElement[rank][j]].neighId[i];
+                        jj++;
+                    }
+                }
+                else
+                    sendNeighIds[rank][j] = sstree.localElements[recvElement[rank][j]].src_id;
+            }
+            MPI_Issend(sendValue[rank], nbRecvElement[rank], MPI_DOUBLE, rank, 0, communicator, &sendRequest[nbSendRequest]);
+            nbSendRequest++;
+            MPI_Issend(sendArea[rank], nbRecvElement[rank], MPI_DOUBLE, rank, 1, communicator, &sendRequest[nbSendRequest]);
+            nbSendRequest++;
+            if (order == 2)
+            {
+                MPI_Issend(sendGrad[rank], 3*nbRecvElement[rank]*(NMAX+1),
+                        MPI_DOUBLE, rank, 2, communicator, &sendRequest[nbSendRequest]);
+                nbSendRequest++;
+                MPI_Issend(sendNeighIds[rank], 4*nbRecvElement[rank]*(NMAX+1), MPI_INT, rank, 3, communicator, &sendRequest[nbSendRequest]);
+                //ym  --> attention taille GloId
+                nbSendRequest++;
+            }
+            else
+            {
+                MPI_Issend(sendNeighIds[rank], 4*nbRecvElement[rank], MPI_INT, rank, 4, communicator, &sendRequest[nbSendRequest]);
+                //ym  --> attention taille GloId
+                nbSendRequest++;
+            }
+        }
+        if (nbSendElement[rank] > 0)
+        {
+            MPI_Irecv(recvValue[rank],  nbSendElement[rank], MPI_DOUBLE, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+            nbRecvRequest++;
+            MPI_Irecv(recvArea[rank],  nbSendElement[rank], MPI_DOUBLE, rank, 1, communicator, &recvRequest[nbRecvRequest]);
+            nbRecvRequest++;
+            if (order == 2)
+            {
+                MPI_Irecv(recvGrad[rank], 3*nbSendElement[rank]*(NMAX+1),
+                        MPI_DOUBLE, rank, 2, communicator, &recvRequest[nbRecvRequest]);
+                nbRecvRequest++;
+                MPI_Irecv(recvNeighIds[rank], 4*nbSendElement[rank]*(NMAX+1), MPI_INT, rank, 3, communicator, &recvRequest[nbRecvRequest]);
+                //ym  --> attention taille GloId
+                nbRecvRequest++;
+            }
+            else
+            {
+                MPI_Irecv(recvNeighIds[rank], 4*nbSendElement[rank], MPI_INT, rank, 4, communicator, &recvRequest[nbRecvRequest]);
+                //ym  --> attention taille GloId
+                nbRecvRequest++;
+            }
+        }
+    }
+    MPI_Waitall(nbRecvRequest, recvRequest, status);
+    MPI_Waitall(nbSendRequest, sendRequest, status);
+    /* now that all values and gradients are available use them to computed interpolated values on target
+       and also to compute weights */
+    int i = 0;
+    for (int j = 0; j < nbElements; j++)
+    {
+        Elt& e = elements[j];
+        /* since for the 2nd order case source grid elements can contribute to a destination grid element over several "paths"
+           (step1: gradient is computed using neighbours on same grid, step2: intersection uses several elements on other grid)
+           accumulate them so that there is only one final weight between two elements */
+        map<GloId,double> wgt_map;
+        /* for destination element `e` loop over all intersetions/the corresponding source elements */
+        for (list<Polyg *>::iterator it = e.is.begin(); it != e.is.end(); it++)
+        {
+            /* it is the intersection element, so it->x and it->area are barycentre and area of intersection element (super mesh)
+               but it->id is id of the source element that it intersects */
+            int n1 = (*it)->id.ind;
+            int rank = (*it)->id.rank;
+            double fk = recvValue[rank][n1];
+            double srcArea = recvArea[rank][n1];
+            double w = (*it)->area;
+            if (quantity) w/=srcArea ;
+            /* first order: src value times weight (weight = supermesh area), later divide by target area */
+            int kk = (order == 2) ? n1 * (NMAX + 1) : n1;
+            GloId neighID = recvNeighIds[rank][kk];
+            wgt_map[neighID] += w;
+            if (order == 2)
+            {
+                for (int k = 0; k < NMAX+1; k++)
+                {
+                    int kk = n1 * (NMAX + 1) + k;
+                    GloId neighID = recvNeighIds[rank][kk];
+                    if (neighID.ind != -1)  wgt_map[neighID] += w * scalarprod(recvGrad[rank][kk], (*it)->x);
+                }
+            }
+        }
+        double renorm=0;
+        if (renormalize)
+            for (map<GloId,double>::iterator it = wgt_map.begin(); it != wgt_map.end(); it++) renorm+=it->second / e.area;
+        else renorm=1. ;
+        for (map<GloId,double>::iterator it = wgt_map.begin(); it != wgt_map.end(); it++)
+        {
+            if (quantity)  this->remapMatrix[i] = (it->second ) / renorm;
+            else this->remapMatrix[i] = (it->second / e.area) / renorm;
+            this->srcAddress[i] = it->first.ind;
+            this->srcRank[i] = it->first.rank;
+            this->dstAddress[i] = j;
+            this->sourceWeightId[i]= it->first.globalId ;
+            this->targetWeightId[i]= targetGlobalId[j] ;
+            i++;
+        }
+    }
+    /* free all memory allocated in this function */
+    for (int rank = 0; rank < mpiSize; rank++)
+    {
+        if (nbSendElement[rank] > 0)
+        {
+            delete[] sendElement[rank];
+            delete[] recvValue[rank];
+            delete[] recvArea[rank];
+            if (order == 2)
+            {
+                delete[] recvGrad[rank];
+            }
+            delete[] recvNeighIds[rank];
+        }
+        if (nbRecvElement[rank] > 0)
+        {
+            delete[] recvElement[rank];
+            delete[] sendValue[rank];
+            delete[] sendArea[rank];
+            if (order == 2)
+                delete[] sendGrad[rank];
+            delete[] sendNeighIds[rank];
+        }
+    }
+    delete[] status;
+    delete[] sendRequest;
+    delete[] recvRequest;
+    delete[] elementList;
+    delete[] nbSendElement;
+    delete[] nbRecvElement;
+    delete[] sendElement;
+    delete[] recvElement;
+    delete[] sendValue;
+    delete[] recvValue;
+    delete[] sendGrad;
+    delete[] recvGrad;
+    delete[] sendNeighIds;
+    delete[] recvNeighIds;
+    return i;
+        int mpiSize, mpiRank;
+        MPI_Comm_size(communicator, &mpiSize);
+        MPI_Comm_rank(communicator, &mpiRank);
+        /* create list of intersections (super mesh elements) for each rank */
+        multimap<int, Polyg *> *elementList = new multimap<int, Polyg *>[mpiSize];
+        for (int j = 0; j < nbElements; j++)
+        {
+                Elt& e = elements[j];
+                for (list<Polyg *>::iterator it = e.is.begin(); it != e.is.end(); it++)
+                        elementList[(*it)->id.rank].insert(pair<int, Polyg *>((*it)->id.ind, *it));
+        }
+        int *nbSendElement = new int[mpiSize];
+        int **sendElement = new int*[mpiSize]; /* indices of elements required from other rank */
+        double **recvValue = new double*[mpiSize];
+        double **recvArea = new double*[mpiSize];
+        Coord **recvGrad = new Coord*[mpiSize];
+        GloId **recvNeighIds = new GloId*[mpiSize]; /* ids of the of the source neighbours which also contribute through gradient */
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                /* get size for allocation */
+                int last = -1; /* compares unequal to any index */
+                int index = -1; /* increased to starting index 0 in first iteration */
+                for (multimap<int, Polyg *>::iterator it = elementList[rank].begin(); it != elementList[rank].end(); ++it)
+                {
+                        if (last != it->first)
+                                index++;
+                        (it->second)->id.ind = index;
+                        last = it->first;
+                }
+                nbSendElement[rank] = index + 1;
+                /* if size is non-zero allocate and collect indices of elements on other ranks that we intersect */
+                if (nbSendElement[rank] > 0)
+                {
+                        sendElement[rank] = new int[nbSendElement[rank]];
+                        recvValue[rank]   = new double[nbSendElement[rank]];
+                        recvArea[rank]    = new double[nbSendElement[rank]];
+                        if (order == 2)
+                        {
+                                recvNeighIds[rank] = new GloId[nbSendElement[rank]*(NMAX+1)];
+                                recvGrad[rank]    = new Coord[nbSendElement[rank]*(NMAX+1)];
+                        }
+                        else
+                                recvNeighIds[rank] = new GloId[nbSendElement[rank]];
+                        last = -1;
+                        index = -1;
+                        for (multimap<int, Polyg *>::iterator it = elementList[rank].begin(); it != elementList[rank].end(); ++it)
+                        {
+                                if (last != it->first)
+                                        index++;
+                                sendElement[rank][index] = it->first;
+                                last = it->first;
+                        }
+                }
+        }
+        /* communicate sizes of source elements to be sent (index lists and later values and gradients) */
+        int *nbRecvElement = new int[mpiSize];
+        MPI_Alltoall(nbSendElement, 1, MPI_INT, nbRecvElement, 1, MPI_INT, communicator);
+        /* communicate indices of source elements on other ranks whoes value and gradient we need (since intersection) */
+        int nbSendRequest = 0;
+        int nbRecvRequest = 0;
+        int **recvElement = new int*[mpiSize];
+        double **sendValue = new double*[mpiSize];
+        double **sendArea = new double*[mpiSize];
+        Coord **sendGrad = new Coord*[mpiSize];
+        GloId **sendNeighIds = new GloId*[mpiSize];
+        MPI_Request *sendRequest = new MPI_Request[4*mpiSize];
+        MPI_Request *recvRequest = new MPI_Request[4*mpiSize];
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbSendElement[rank] > 0)
+                {
+                        MPI_Issend(sendElement[rank], nbSendElement[rank], MPI_INT, rank, 0, communicator, &sendRequest[nbSendRequest]);
+                        nbSendRequest++;
+                }
+                if (nbRecvElement[rank] > 0)
+                {
+                        recvElement[rank] = new int[nbRecvElement[rank]];
+                        sendValue[rank]   = new double[nbRecvElement[rank]];
+                        sendArea[rank]   = new double[nbRecvElement[rank]];
+                        if (order == 2)
+                        {
+                                sendNeighIds[rank] = new GloId[nbRecvElement[rank]*(NMAX+1)];
+                                sendGrad[rank]    = new Coord[nbRecvElement[rank]*(NMAX+1)];
+                        }
+                        else
+                        {
+                                sendNeighIds[rank] = new GloId[nbRecvElement[rank]];
+                        }
+                        MPI_Irecv(recvElement[rank], nbRecvElement[rank], MPI_INT, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+                        nbRecvRequest++;
+                }
+        }
+  MPI_Status *status = new MPI_Status[4*mpiSize];
+  MPI_Waitall(nbSendRequest, sendRequest, status);
+  MPI_Waitall(nbRecvRequest, recvRequest, status);
+        /* for all indices that have been received from requesting ranks: pack values and gradients, then send */
+        nbSendRequest = 0;
+        nbRecvRequest = 0;
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbRecvElement[rank] > 0)
+                {
+                        int jj = 0; // jj == j if no weight writing
+                        for (int j = 0; j < nbRecvElement[rank]; j++)
+                        {
+                                sendValue[rank][j] = sstree.localElements[recvElement[rank][j]].val;
+                                sendArea[rank][j] = sstree.localElements[recvElement[rank][j]].area;
+                                if (order == 2)
+                                {
+                                        sendGrad[rank][jj] = sstree.localElements[recvElement[rank][j]].grad;
+                                        sendNeighIds[rank][jj] = sstree.localElements[recvElement[rank][j]].src_id;
+                                        jj++;
+                                        for (int i = 0; i < NMAX; i++)
+                                        {
+                                                sendGrad[rank][jj] = sstree.localElements[recvElement[rank][j]].gradNeigh[i];
+            sendNeighIds[rank][jj] = sstree.localElements[recvElement[rank][j]].neighId[i];
+                                                jj++;
+                                        }
+                                }
+                                else
+                                        sendNeighIds[rank][j] = sstree.localElements[recvElement[rank][j]].src_id;
+                        }
+                        MPI_Issend(sendValue[rank],  nbRecvElement[rank], MPI_DOUBLE, rank, 0, communicator, &sendRequest[nbSendRequest]);
+                        nbSendRequest++;
+                        MPI_Issend(sendArea[rank],  nbRecvElement[rank], MPI_DOUBLE, rank, 0, communicator, &sendRequest[nbSendRequest]);
+                        nbSendRequest++;
+                        if (order == 2)
+                        {
+                                MPI_Issend(sendGrad[rank], 3*nbRecvElement[rank]*(NMAX+1),
+                                                                MPI_DOUBLE, rank, 0, communicator, &sendRequest[nbSendRequest]);
+                                nbSendRequest++;
+                                MPI_Issend(sendNeighIds[rank], 4*nbRecvElement[rank]*(NMAX+1), MPI_INT, rank, 0, communicator, &sendRequest[nbSendRequest]);
+//ym  --> attention taille GloId
+                                nbSendRequest++;
+                        }
+                        else
+                        {
+                                MPI_Issend(sendNeighIds[rank], 4*nbRecvElement[rank], MPI_INT, rank, 0, communicator, &sendRequest[nbSendRequest]);
+//ym  --> attention taille GloId
+                                nbSendRequest++;
+                        }
+                }
+                if (nbSendElement[rank] > 0)
+                {
+                        MPI_Irecv(recvValue[rank],  nbSendElement[rank], MPI_DOUBLE, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+                        nbRecvRequest++;
+                        MPI_Irecv(recvArea[rank],  nbSendElement[rank], MPI_DOUBLE, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+                        nbRecvRequest++;
+                        if (order == 2)
+                        {
+                                MPI_Irecv(recvGrad[rank], 3*nbSendElement[rank]*(NMAX+1),
+                                                MPI_DOUBLE, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+                                nbRecvRequest++;
+                                MPI_Irecv(recvNeighIds[rank], 4*nbSendElement[rank]*(NMAX+1), MPI_INT, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+//ym  --> attention taille GloId
+                                nbRecvRequest++;
+                        }
+                        else
+                        {
+                                MPI_Irecv(recvNeighIds[rank], 4*nbSendElement[rank], MPI_INT, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+//ym  --> attention taille GloId
+                                nbRecvRequest++;
+                        }
+                }
+        }
+        MPI_Waitall(nbSendRequest, sendRequest, status);
+        MPI_Waitall(nbRecvRequest, recvRequest, status);
+        /* now that all values and gradients are available use them to computed interpolated values on target
+           and also to compute weights */
+        int i = 0;
+        for (int j = 0; j < nbElements; j++)
+        {
+                Elt& e = elements[j];
+                /* since for the 2nd order case source grid elements can contribute to a destination grid element over several "paths"
+                   (step1: gradient is computed using neighbours on same grid, step2: intersection uses several elements on other grid)
+                   accumulate them so that there is only one final weight between two elements */
+                map<GloId,double> wgt_map;
+                /* for destination element `e` loop over all intersetions/the corresponding source elements */
+                for (list<Polyg *>::iterator it = e.is.begin(); it != e.is.end(); it++)
+                {
+                        /* it is the intersection element, so it->x and it->area are barycentre and area of intersection element (super mesh)
+                        but it->id is id of the source element that it intersects */
+                        int n1 = (*it)->id.ind;
+                        int rank = (*it)->id.rank;
+                        double fk = recvValue[rank][n1];
+                        double srcArea = recvArea[rank][n1];
+                        double w = (*it)->area;
+      if (quantity) w/=srcArea ;
+                        /* first order: src value times weight (weight = supermesh area), later divide by target area */
+                        int kk = (order == 2) ? n1 * (NMAX + 1) : n1;
+                        GloId neighID = recvNeighIds[rank][kk];
+                        wgt_map[neighID] += w;
+                        if (order == 2)
+                        {
+                                for (int k = 0; k < NMAX+1; k++)
+                                {
+                                        int kk = n1 * (NMAX + 1) + k;
+                                        GloId neighID = recvNeighIds[rank][kk];
+                                        if (neighID.ind != -1)  wgt_map[neighID] += w * scalarprod(recvGrad[rank][kk], (*it)->x);
+                                }
+                        }
+                }
+    double renorm=0;
+    if (renormalize)
+      for (map<GloId,double>::iterator it = wgt_map.begin(); it != wgt_map.end(); it++) renorm+=it->second / e.area;
+    else renorm=1. ;
+    for (map<GloId,double>::iterator it = wgt_map.begin(); it != wgt_map.end(); it++)
+                {
+      if (quantity)  this->remapMatrix[i] = (it->second ) / renorm;
+                        else this->remapMatrix[i] = (it->second / e.area) / renorm;
+                        this->srcAddress[i] = it->first.ind;
+                        this->srcRank[i] = it->first.rank;
+                        this->dstAddress[i] = j;
+      this->sourceWeightId[i]= it->first.globalId ;
+      this->targetWeightId[i]= targetGlobalId[j] ;
+                        i++;
+                }
+        }
+        /* free all memory allocated in this function */
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbSendElement[rank] > 0)
+                {
+                        delete[] sendElement[rank];
+                        delete[] recvValue[rank];
+                        delete[] recvArea[rank];
+                        if (order == 2)
+                        {
+                                delete[] recvGrad[rank];
+                        }
+                        delete[] recvNeighIds[rank];
+                }
+                if (nbRecvElement[rank] > 0)
+                {
+                        delete[] recvElement[rank];
+                        delete[] sendValue[rank];
+                        delete[] sendArea[rank];
+                        if (order == 2)
+                                delete[] sendGrad[rank];
+                        delete[] sendNeighIds[rank];
+                }
+        }
+        delete[] status;
+        delete[] sendRequest;
+        delete[] recvRequest;
+        delete[] elementList;
+        delete[] nbSendElement;
+        delete[] nbRecvElement;
+        delete[] sendElement;
+        delete[] recvElement;
+        delete[] sendValue;
+        delete[] recvValue;
+        delete[] sendGrad;
+        delete[] recvGrad;
+        delete[] sendNeighIds;
+        delete[] recvNeighIds;
+        return i;
+}
 void Mapper::computeGrads()
+{
     /* array of pointers to collect local elements and elements received from other cpu */
     vector<Elt*> globalElements(sstree.nbLocalElements + nbNeighbourElements);
     int index = 0;
     for (int i = 0; i < sstree.nbLocalElements; i++, index++)
         globalElements[index] = &(sstree.localElements[i]);
     for (int i = 0; i < nbNeighbourElements; i++, index++)
         globalElements[index] = &neighbourElements[i];
     update_baryc(sstree.localElements, sstree.nbLocalElements);
     computeGradients(&globalElements[0], sstree.nbLocalElements);
+        /* array of pointers to collect local elements and elements received from other cpu */
+        vector<Elt*> globalElements(sstree.nbLocalElements + nbNeighbourElements);
+        int index = 0;
+        for (int i = 0; i < sstree.nbLocalElements; i++, index++)
+                globalElements[index] = &(sstree.localElements[i]);
+        for (int i = 0; i < nbNeighbourElements; i++, index++)
+                globalElements[index] = &neighbourElements[i];
+        update_baryc(sstree.localElements, sstree.nbLocalElements);
+        computeGradients(&globalElements[0], sstree.nbLocalElements);
+}
 /** for each element of the source grid, finds all the neighbouring elements that share an edge
   (filling array neighbourElements). This is used later to compute gradients */
+    (filling array neighbourElements). This is used later to compute gradients */
 void Mapper::buildMeshTopology()
+{
     int mpiSize, mpiRank;
     MPI_Comm_size(communicator, &mpiSize);
     MPI_Comm_rank(communicator, &mpiRank);
     vector<Node> *routingList = new vector<Node>[mpiSize];
     vector<vector<int> > routes(sstree.localTree.leafs.size());
     sstree.routeIntersections(routes, sstree.localTree.leafs);
     for (int i = 0; i < routes.size(); ++i)
         for (int k = 0; k < routes[i].size(); ++k)
             routingList[routes[i][k]].push_back(sstree.localTree.leafs[i]);
     routingList[mpiRank].clear();
     CMPIRouting mpiRoute(communicator);
     mpiRoute.init(routes);
     int nRecv = mpiRoute.getTotalSourceElement();
     int *nbSendNode = new int[mpiSize];
     int *nbRecvNode = new int[mpiSize];
     int *sendMessageSize = new int[mpiSize];
     int *recvMessageSize = new int[mpiSize];
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         nbSendNode[rank] = routingList[rank].size();
         sendMessageSize[rank] = 0;
         for (size_t j = 0; j < routingList[rank].size(); j++)
+        {
             Elt *elt = (Elt *) (routingList[rank][j].data);
             sendMessageSize[rank] += packedPolygonSize(*elt);
+        }
+    }
     MPI_Alltoall(nbSendNode, 1, MPI_INT, nbRecvNode, 1, MPI_INT, communicator);
     MPI_Alltoall(sendMessageSize, 1, MPI_INT, recvMessageSize, 1, MPI_INT, communicator);
     char **sendBuffer = new char*[mpiSize];
     char **recvBuffer = new char*[mpiSize];
     int *pos = new int[mpiSize];
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         if (nbSendNode[rank] > 0) sendBuffer[rank] = new char[sendMessageSize[rank]];
         if (nbRecvNode[rank] > 0) recvBuffer[rank] = new char[recvMessageSize[rank]];
+    }
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         pos[rank] = 0;
         for (size_t j = 0; j < routingList[rank].size(); j++)
+        {
             Elt *elt = (Elt *) (routingList[rank][j].data);
             packPolygon(*elt, sendBuffer[rank], pos[rank]);
+        }
+    }
     delete [] routingList;
     int nbSendRequest = 0;
     int nbRecvRequest = 0;
     MPI_Request *sendRequest = new MPI_Request[mpiSize];
     MPI_Request *recvRequest = new MPI_Request[mpiSize];
     MPI_Status  *status      = new MPI_Status[mpiSize];
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         if (nbSendNode[rank] > 0)
+        {
             MPI_Issend(sendBuffer[rank], sendMessageSize[rank], MPI_CHAR, rank, 0, communicator, &sendRequest[nbSendRequest]);
             nbSendRequest++;
+        }
         if (nbRecvNode[rank] > 0)
+        {
             MPI_Irecv(recvBuffer[rank], recvMessageSize[rank], MPI_CHAR, rank, 0, communicator, &recvRequest[nbRecvRequest]);
             nbRecvRequest++;
+        }
+    }
     MPI_Waitall(nbRecvRequest, recvRequest, status);
     MPI_Waitall(nbSendRequest, sendRequest, status);
     for (int rank = 0; rank < mpiSize; rank++)
         if (nbSendNode[rank] > 0) delete [] sendBuffer[rank];
     delete [] sendBuffer;
     char **sendBuffer2 = new char*[mpiSize];
     char **recvBuffer2 = new char*[mpiSize];
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         nbSendNode[rank] = 0;
         sendMessageSize[rank] = 0;
         if (nbRecvNode[rank] > 0)
+        {
             set<NodePtr> neighbourList;
             pos[rank] = 0;
             for (int j = 0; j < nbRecvNode[rank]; j++)
+            {
                 Elt elt;
                 unpackPolygon(elt, recvBuffer[rank], pos[rank]);
                 Node node(elt.x, cptRadius(elt), &elt);
                 findNeighbour(sstree.localTree.root, &node, neighbourList);
+            }
             nbSendNode[rank] = neighbourList.size();
             for (set<NodePtr>::iterator it = neighbourList.begin(); it != neighbourList.end(); it++)
+            {
                 Elt *elt = (Elt *) ((*it)->data);
                 sendMessageSize[rank] += packedPolygonSize(*elt);
+            }
             sendBuffer2[rank] = new char[sendMessageSize[rank]];
             pos[rank] = 0;
             for (set<NodePtr>::iterator it = neighbourList.begin(); it != neighbourList.end(); it++)
+            {
                 Elt *elt = (Elt *) ((*it)->data);
                 packPolygon(*elt, sendBuffer2[rank], pos[rank]);
+            }
+        }
+    }
     for (int rank = 0; rank < mpiSize; rank++)
         if (nbRecvNode[rank] > 0) delete [] recvBuffer[rank];
     delete [] recvBuffer;
     MPI_Barrier(communicator);
     MPI_Alltoall(nbSendNode, 1, MPI_INT, nbRecvNode, 1, MPI_INT, communicator);
     MPI_Alltoall(sendMessageSize, 1, MPI_INT, recvMessageSize, 1, MPI_INT, communicator);
     for (int rank = 0; rank < mpiSize; rank++)
         if (nbRecvNode[rank] > 0) recvBuffer2[rank] = new char[recvMessageSize[rank]];
     nbSendRequest = 0;
     nbRecvRequest = 0;
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         if (nbSendNode[rank] > 0)
+        {
             MPI_Issend(sendBuffer2[rank], sendMessageSize[rank], MPI_CHAR, rank, 0, communicator, &sendRequest[nbSendRequest]);
             nbSendRequest++;
+        }
         if (nbRecvNode[rank] > 0)
+        {
             MPI_Irecv(recvBuffer2[rank], recvMessageSize[rank], MPI_CHAR, rank, 0, communicator, &recvRequest[nbRecvRequest]);
             nbRecvRequest++;
+        }
+    }
     MPI_Waitall(nbRecvRequest, recvRequest, status);
     MPI_Waitall(nbSendRequest, sendRequest, status);
     int nbNeighbourNodes = 0;
     for (int rank = 0; rank < mpiSize; rank++)
         nbNeighbourNodes += nbRecvNode[rank];
     neighbourElements = new Elt[nbNeighbourNodes];
     nbNeighbourElements = nbNeighbourNodes;
     int index = 0;
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         pos[rank] = 0;
         for (int j = 0; j < nbRecvNode[rank]; j++)
+        {
             unpackPolygon(neighbourElements[index], recvBuffer2[rank], pos[rank]);
             neighbourElements[index].id.ind = sstree.localTree.leafs.size() + index;
             index++;
+        }
+    }
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         if (nbRecvNode[rank] > 0) delete [] recvBuffer2[rank];
         if (nbSendNode[rank] > 0) delete [] sendBuffer2[rank];
+    }
     delete [] recvBuffer2;
     delete [] sendBuffer2;
     delete [] sendMessageSize;
     delete [] recvMessageSize;
     delete [] nbSendNode;
     delete [] nbRecvNode;
     delete [] sendRequest;
     delete [] recvRequest;
     delete [] status;
     delete [] pos;
     /* re-compute on received elements to avoid having to send this information */
     neighbourNodes.resize(nbNeighbourNodes);
     setCirclesAndLinks(neighbourElements, neighbourNodes);
     cptAllEltsGeom(neighbourElements, nbNeighbourNodes, srcGrid.pole);
     /* the local SS tree must include nodes from other cpus if they are potential
        intersector of a local node */
     sstree.localTree.insertNodes(neighbourNodes);
     /* for every local element,
        use the SS-tree to find all elements (including neighbourElements)
        who are potential neighbours because their circles intersect,
        then check all canditates for common edges to build up connectivity information
        */
     for (int j = 0; j < sstree.localTree.leafs.size(); j++)
+    {
         Node& node = sstree.localTree.leafs[j];
         /* find all leafs whoes circles that intersect node's circle and save into node->intersectors */
         node.search(sstree.localTree.root);
         Elt *elt = (Elt *)(node.data);
         for (int i = 0; i < elt->n; i++) elt->neighbour[i] = NOT_FOUND;
         /* for element `elt` loop through all nodes in the SS-tree
            whoes circles intersect with the circle around `elt` (the SS intersectors)
            and check if they are neighbours in the sense that the two elements share an edge.
            If they do, save this information for elt */
         for (list<NodePtr>::iterator it = (node.intersectors).begin(); it != (node.intersectors).end(); ++it)
+        {
             Elt *elt2 = (Elt *)((*it)->data);
             set_neighbour(*elt, *elt2);
+        }
+    }
+        int mpiSize, mpiRank;
+        MPI_Comm_size(communicator, &mpiSize);
+        MPI_Comm_rank(communicator, &mpiRank);
+        vector<Node> *routingList = new vector<Node>[mpiSize];
+        vector<vector<int> > routes(sstree.localTree.leafs.size());
+        sstree.routeIntersections(routes, sstree.localTree.leafs);
+        for (int i = 0; i < routes.size(); ++i)
+                for (int k = 0; k < routes[i].size(); ++k)
+                        routingList[routes[i][k]].push_back(sstree.localTree.leafs[i]);
+        routingList[mpiRank].clear();
+        CMPIRouting mpiRoute(communicator);
+        mpiRoute.init(routes);
+        int nRecv = mpiRoute.getTotalSourceElement();
+        int *nbSendNode = new int[mpiSize];
+        int *nbRecvNode = new int[mpiSize];
+        int *sendMessageSize = new int[mpiSize];
+        int *recvMessageSize = new int[mpiSize];
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                nbSendNode[rank] = routingList[rank].size();
+                sendMessageSize[rank] = 0;
+                for (size_t j = 0; j < routingList[rank].size(); j++)
+                {
+                        Elt *elt = (Elt *) (routingList[rank][j].data);
+                        sendMessageSize[rank] += packedPolygonSize(*elt);
+                }
+        }
+        MPI_Alltoall(nbSendNode, 1, MPI_INT, nbRecvNode, 1, MPI_INT, communicator);
+        MPI_Alltoall(sendMessageSize, 1, MPI_INT, recvMessageSize, 1, MPI_INT, communicator);
+        char **sendBuffer = new char*[mpiSize];
+        char **recvBuffer = new char*[mpiSize];
+        int *pos = new int[mpiSize];
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbSendNode[rank] > 0) sendBuffer[rank] = new char[sendMessageSize[rank]];
+                if (nbRecvNode[rank] > 0) recvBuffer[rank] = new char[recvMessageSize[rank]];
+        }
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                pos[rank] = 0;
+                for (size_t j = 0; j < routingList[rank].size(); j++)
+                {
+                        Elt *elt = (Elt *) (routingList[rank][j].data);
+                        packPolygon(*elt, sendBuffer[rank], pos[rank]);
+                }
+        }
+        delete [] routingList;
+        int nbSendRequest = 0;
+        int nbRecvRequest = 0;
+        MPI_Request *sendRequest = new MPI_Request[mpiSize];
+        MPI_Request *recvRequest = new MPI_Request[mpiSize];
+        MPI_Status  *status      = new MPI_Status[mpiSize];
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbSendNode[rank] > 0)
+                {
+                        MPI_Issend(sendBuffer[rank], sendMessageSize[rank], MPI_CHAR, rank, 0, communicator, &sendRequest[nbSendRequest]);
+                        nbSendRequest++;
+                }
+                if (nbRecvNode[rank] > 0)
+                {
+                        MPI_Irecv(recvBuffer[rank], recvMessageSize[rank], MPI_CHAR, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+                        nbRecvRequest++;
+                }
+        }
+  MPI_Waitall(nbRecvRequest, recvRequest, status);
+  MPI_Waitall(nbSendRequest, sendRequest, status);
+        for (int rank = 0; rank < mpiSize; rank++)
+                if (nbSendNode[rank] > 0) delete [] sendBuffer[rank];
+        delete [] sendBuffer;
+        char **sendBuffer2 = new char*[mpiSize];
+        char **recvBuffer2 = new char*[mpiSize];
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                nbSendNode[rank] = 0;
+                sendMessageSize[rank] = 0;
+                if (nbRecvNode[rank] > 0)
+                {
+                        set<NodePtr> neighbourList;
+                        pos[rank] = 0;
+                        for (int j = 0; j < nbRecvNode[rank]; j++)
+                        {
+                                Elt elt;
+                                unpackPolygon(elt, recvBuffer[rank], pos[rank]);
+                                Node node(elt.x, cptRadius(elt), &elt);
+                                findNeighbour(sstree.localTree.root, &node, neighbourList);
+                        }
+                        nbSendNode[rank] = neighbourList.size();
+                        for (set<NodePtr>::iterator it = neighbourList.begin(); it != neighbourList.end(); it++)
+                        {
+                                Elt *elt = (Elt *) ((*it)->data);
+                                sendMessageSize[rank] += packedPolygonSize(*elt);
+                        }
+                        sendBuffer2[rank] = new char[sendMessageSize[rank]];
+                        pos[rank] = 0;
+                        for (set<NodePtr>::iterator it = neighbourList.begin(); it != neighbourList.end(); it++)
+                        {
+                                Elt *elt = (Elt *) ((*it)->data);
+                                packPolygon(*elt, sendBuffer2[rank], pos[rank]);
+                        }
+                }
+        }
+        for (int rank = 0; rank < mpiSize; rank++)
+                if (nbRecvNode[rank] > 0) delete [] recvBuffer[rank];
+        delete [] recvBuffer;
+        MPI_Barrier(communicator);
+        MPI_Alltoall(nbSendNode, 1, MPI_INT, nbRecvNode, 1, MPI_INT, communicator);
+        MPI_Alltoall(sendMessageSize, 1, MPI_INT, recvMessageSize, 1, MPI_INT, communicator);
+        for (int rank = 0; rank < mpiSize; rank++)
+                if (nbRecvNode[rank] > 0) recvBuffer2[rank] = new char[recvMessageSize[rank]];
+        nbSendRequest = 0;
+        nbRecvRequest = 0;
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbSendNode[rank] > 0)
+                {
+                        MPI_Issend(sendBuffer2[rank], sendMessageSize[rank], MPI_CHAR, rank, 0, communicator, &sendRequest[nbSendRequest]);
+                        nbSendRequest++;
+                }
+                if (nbRecvNode[rank] > 0)
+                {
+                        MPI_Irecv(recvBuffer2[rank], recvMessageSize[rank], MPI_CHAR, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+                        nbRecvRequest++;
+                }
+        }
+        MPI_Waitall(nbRecvRequest, recvRequest, status);
+        MPI_Waitall(nbSendRequest, sendRequest, status);
+        int nbNeighbourNodes = 0;
+        for (int rank = 0; rank < mpiSize; rank++)
+                nbNeighbourNodes += nbRecvNode[rank];
+        neighbourElements = new Elt[nbNeighbourNodes];
+        nbNeighbourElements = nbNeighbourNodes;
+        int index = 0;
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                pos[rank] = 0;
+                for (int j = 0; j < nbRecvNode[rank]; j++)
+                {
+                        unpackPolygon(neighbourElements[index], recvBuffer2[rank], pos[rank]);
+                        neighbourElements[index].id.ind = sstree.localTree.leafs.size() + index;
+                        index++;
+                }
+        }
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbRecvNode[rank] > 0) delete [] recvBuffer2[rank];
+                if (nbSendNode[rank] > 0) delete [] sendBuffer2[rank];
+        }
+        delete [] recvBuffer2;
+        delete [] sendBuffer2;
+        delete [] sendMessageSize;
+        delete [] recvMessageSize;
+        delete [] nbSendNode;
+        delete [] nbRecvNode;
+        delete [] sendRequest;
+        delete [] recvRequest;
+        delete [] status;
+        delete [] pos;
+        /* re-compute on received elements to avoid having to send this information */
+        neighbourNodes.resize(nbNeighbourNodes);
+        setCirclesAndLinks(neighbourElements, neighbourNodes);
+        cptAllEltsGeom(neighbourElements, nbNeighbourNodes, srcGrid.pole);
+        /* the local SS tree must include nodes from other cpus if they are potential
+           intersector of a local node */
+        sstree.localTree.insertNodes(neighbourNodes);
+        /* for every local element,
+           use the SS-tree to find all elements (including neighbourElements)
+           who are potential neighbours because their circles intersect,
+           then check all canditates for common edges to build up connectivity information
+        */
+        for (int j = 0; j < sstree.localTree.leafs.size(); j++)
+        {
+                Node& node = sstree.localTree.leafs[j];
+                /* find all leafs whoes circles that intersect node's circle and save into node->intersectors */
+                node.search(sstree.localTree.root);
+                Elt *elt = (Elt *)(node.data);
+                for (int i = 0; i < elt->n; i++) elt->neighbour[i] = NOT_FOUND;
+                /* for element `elt` loop through all nodes in the SS-tree
+                   whoes circles intersect with the circle around `elt` (the SS intersectors)
+                   and check if they are neighbours in the sense that the two elements share an edge.
+                   If they do, save this information for elt */
+                for (list<NodePtr>::iterator it = (node.intersectors).begin(); it != (node.intersectors).end(); ++it)
+                {
+                        Elt *elt2 = (Elt *)((*it)->data);
+                        set_neighbour(*elt, *elt2);
+                }
+        }
+}
 …
 void Mapper::computeIntersection(Elt *elements, int nbElements)
+{
     int mpiSize, mpiRank;
     MPI_Comm_size(communicator, &mpiSize);
     MPI_Comm_rank(communicator, &mpiRank);
     MPI_Barrier(communicator);
     vector<Node> *routingList = new vector<Node>[mpiSize];
     vector<Node> routeNodes;  routeNodes.reserve(nbElements);
     for (int j = 0; j < nbElements; j++)
+    {
         elements[j].id.ind = j;
         elements[j].id.rank = mpiRank;
         routeNodes.push_back(Node(elements[j].x, cptRadius(elements[j]), &elements[j]));
+    }
     vector<vector<int> > routes(routeNodes.size());
     sstree.routeIntersections(routes, routeNodes);
     for (int i = 0; i < routes.size(); ++i)
         for (int k = 0; k < routes[i].size(); ++k)
             routingList[routes[i][k]].push_back(routeNodes[i]);
     if (verbose >= 2)
+    {
         cout << " --> rank  " << mpiRank << " nbElements " << nbElements << " : ";
         for (int rank = 0; rank < mpiSize; rank++)
             cout << routingList[rank].size() << "   ";
         cout << endl;
+    }
     MPI_Barrier(communicator);
     int *nbSendNode = new int[mpiSize];
     int *nbRecvNode = new int[mpiSize];
     int *sentMessageSize = new int[mpiSize];
     int *recvMessageSize = new int[mpiSize];
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         nbSendNode[rank] = routingList[rank].size();
         sentMessageSize[rank] = 0;
         for (size_t j = 0; j < routingList[rank].size(); j++)
+        {
             Elt *elt = (Elt *) (routingList[rank][j].data);
             sentMessageSize[rank] += packedPolygonSize(*elt);
+        }
+    }
     MPI_Alltoall(nbSendNode, 1, MPI_INT, nbRecvNode, 1, MPI_INT, communicator);
     MPI_Alltoall(sentMessageSize, 1, MPI_INT, recvMessageSize, 1, MPI_INT, communicator);
     int total = 0;
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         total = total + nbRecvNode[rank];
+    }
     if (verbose >= 2) cout << "---> rank " << mpiRank << " : compute intersection : total received nodes  " << total << endl;
     char **sendBuffer = new char*[mpiSize];
     char **recvBuffer = new char*[mpiSize];
     int *pos = new int[mpiSize];
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         if (nbSendNode[rank] > 0) sendBuffer[rank] = new char[sentMessageSize[rank]];
         if (nbRecvNode[rank] > 0) recvBuffer[rank] = new char[recvMessageSize[rank]];
+    }
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         pos[rank] = 0;
         for (size_t j = 0; j < routingList[rank].size(); j++)
+        {
             Elt* elt = (Elt *) (routingList[rank][j].data);
             packPolygon(*elt, sendBuffer[rank], pos[rank]);
+        }
+    }
     delete [] routingList;
     int nbSendRequest = 0;
     int nbRecvRequest = 0;
     MPI_Request *sendRequest = new MPI_Request[mpiSize];
     MPI_Request *recvRequest = new MPI_Request[mpiSize];
     MPI_Status   *status = new MPI_Status[mpiSize];
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         if (nbSendNode[rank] > 0)
+        {
             MPI_Issend(sendBuffer[rank], sentMessageSize[rank], MPI_CHAR, rank, 0, communicator, &sendRequest[nbSendRequest]);
             nbSendRequest++;
+        }
         if (nbRecvNode[rank] > 0)
+        {
             MPI_Irecv(recvBuffer[rank], recvMessageSize[rank], MPI_CHAR, rank, 0, communicator, &recvRequest[nbRecvRequest]);
             nbRecvRequest++;
+        }
+    }
     MPI_Waitall(nbRecvRequest, recvRequest, status);
     MPI_Waitall(nbSendRequest, sendRequest, status);
     char **sendBuffer2 = new char*[mpiSize];
     char **recvBuffer2 = new char*[mpiSize];
     double tic = cputime();
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         sentMessageSize[rank] = 0;
         if (nbRecvNode[rank] > 0)
+        {
             Elt *recvElt = new Elt[nbRecvNode[rank]];
             pos[rank] = 0;
             for (int j = 0; j < nbRecvNode[rank]; j++)
+            {
                 unpackPolygon(recvElt[j], recvBuffer[rank], pos[rank]);
                 cptEltGeom(recvElt[j], tgtGrid.pole);
                 Node recvNode(recvElt[j].x, cptRadius(recvElt[j]), &recvElt[j]);
                 recvNode.search(sstree.localTree.root);
                 /* for a node holding an element of the target, loop throught candidates for intersecting source */
                 for (list<NodePtr>::iterator it = (recvNode.intersectors).begin(); it != (recvNode.intersectors).end(); ++it)
+                {
                     Elt *elt2 = (Elt *) ((*it)->data);
                     /* recvElt is target, elt2 is source */
                     intersect(&recvElt[j], elt2);
                     //intersect_ym(&recvElt[j], elt2);
+                }
                 if (recvElt[j].is.size() > 0) sentMessageSize[rank] += packIntersectionSize(recvElt[j]);
                 // here recvNode goes out of scope
+            }
             if (sentMessageSize[rank] > 0)
+            {
                 sentMessageSize[rank] += sizeof(int);
                 sendBuffer2[rank] = new char[sentMessageSize[rank]];
                 *((int *) sendBuffer2[rank]) = 0;
                 pos[rank] = sizeof(int);
                 for (int j = 0; j < nbRecvNode[rank]; j++)
+                {
                     packIntersection(recvElt[j], sendBuffer2[rank], pos[rank]);
                     //FIXME should be deleted: recvElt[j].delete_intersections(); // intersection areas have been packed to buffer and won't be used any more
+                }
+            }
             delete [] recvElt;
+        }
+    }
     delete [] pos;
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         if (nbSendNode[rank] > 0) delete [] sendBuffer[rank];
         if (nbRecvNode[rank] > 0) delete [] recvBuffer[rank];
         nbSendNode[rank] = 0;
+    }
     if (verbose >= 2) cout << "Rank " << mpiRank << "  Compute (internal) intersection " << cputime() - tic << " s" << endl;
     MPI_Alltoall(sentMessageSize, 1, MPI_INT, recvMessageSize, 1, MPI_INT, communicator);
     for (int rank = 0; rank < mpiSize; rank++)
         if (recvMessageSize[rank] > 0)
             recvBuffer2[rank] = new char[recvMessageSize[rank]];
     nbSendRequest = 0;
     nbRecvRequest = 0;
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         if (sentMessageSize[rank] > 0)
+        {
             MPI_Issend(sendBuffer2[rank], sentMessageSize[rank], MPI_CHAR, rank, 0, communicator, &sendRequest[nbSendRequest]);
             nbSendRequest++;
+        }
         if (recvMessageSize[rank] > 0)
+        {
             MPI_Irecv(recvBuffer2[rank], recvMessageSize[rank], MPI_CHAR, rank, 0, communicator, &recvRequest[nbRecvRequest]);
             nbRecvRequest++;
+        }
+    }
     MPI_Waitall(nbRecvRequest, recvRequest, status);
     MPI_Waitall(nbSendRequest, sendRequest, status);
     delete [] sendRequest;
     delete [] recvRequest;
     delete [] status;
     for (int rank = 0; rank < mpiSize; rank++)
+    {
         if (nbRecvNode[rank] > 0)
+        {
             if (sentMessageSize[rank] > 0)
                 delete [] sendBuffer2[rank];
+        }
         if (recvMessageSize[rank] > 0)
+        {
             unpackIntersection(elements, recvBuffer2[rank]);
             delete [] recvBuffer2[rank];
+        }
+    }
     delete [] sendBuffer2;
     delete [] recvBuffer2;
     delete [] sendBuffer;
     delete [] recvBuffer;
     delete [] nbSendNode;
     delete [] nbRecvNode;
     delete [] sentMessageSize;
     delete [] recvMessageSize;
+        int mpiSize, mpiRank;
+        MPI_Comm_size(communicator, &mpiSize);
+        MPI_Comm_rank(communicator, &mpiRank);
+        MPI_Barrier(communicator);
+        vector<Node> *routingList = new vector<Node>[mpiSize];
+        vector<Node> routeNodes;  routeNodes.reserve(nbElements);
+        for (int j = 0; j < nbElements; j++)
+        {
+                elements[j].id.ind = j;
+                elements[j].id.rank = mpiRank;
+                routeNodes.push_back(Node(elements[j].x, cptRadius(elements[j]), &elements[j]));
+        }
+        vector<vector<int> > routes(routeNodes.size());
+        sstree.routeIntersections(routes, routeNodes);
+        for (int i = 0; i < routes.size(); ++i)
+                for (int k = 0; k < routes[i].size(); ++k)
+                        routingList[routes[i][k]].push_back(routeNodes[i]);
+        if (verbose >= 2)
+        {
+                cout << " --> rank  " << mpiRank << " nbElements " << nbElements << " : ";
+                for (int rank = 0; rank < mpiSize; rank++)
+                        cout << routingList[rank].size() << "   ";
+                cout << endl;
+        }
+        MPI_Barrier(communicator);
+        int *nbSendNode = new int[mpiSize];
+        int *nbRecvNode = new int[mpiSize];
+        int *sentMessageSize = new int[mpiSize];
+        int *recvMessageSize = new int[mpiSize];
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                nbSendNode[rank] = routingList[rank].size();
+                sentMessageSize[rank] = 0;
+                for (size_t j = 0; j < routingList[rank].size(); j++)
+                {
+                        Elt *elt = (Elt *) (routingList[rank][j].data);
+                        sentMessageSize[rank] += packedPolygonSize(*elt);
+                }
+        }
+        MPI_Alltoall(nbSendNode, 1, MPI_INT, nbRecvNode, 1, MPI_INT, communicator);
+        MPI_Alltoall(sentMessageSize, 1, MPI_INT, recvMessageSize, 1, MPI_INT, communicator);
+        int total = 0;
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                total = total + nbRecvNode[rank];
+        }
+        if (verbose >= 2) cout << "---> rank " << mpiRank << " : compute intersection : total received nodes  " << total << endl;
+        char **sendBuffer = new char*[mpiSize];
+        char **recvBuffer = new char*[mpiSize];
+        int *pos = new int[mpiSize];
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbSendNode[rank] > 0) sendBuffer[rank] = new char[sentMessageSize[rank]];
+                if (nbRecvNode[rank] > 0) recvBuffer[rank] = new char[recvMessageSize[rank]];
+        }
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                pos[rank] = 0;
+                for (size_t j = 0; j < routingList[rank].size(); j++)
+                {
+                        Elt* elt = (Elt *) (routingList[rank][j].data);
+                        packPolygon(*elt, sendBuffer[rank], pos[rank]);
+                }
+        }
+        delete [] routingList;
+        int nbSendRequest = 0;
+        int nbRecvRequest = 0;
+        MPI_Request *sendRequest = new MPI_Request[mpiSize];
+        MPI_Request *recvRequest = new MPI_Request[mpiSize];
+        MPI_Status   *status = new MPI_Status[mpiSize];
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbSendNode[rank] > 0)
+                {
+                        MPI_Issend(sendBuffer[rank], sentMessageSize[rank], MPI_CHAR, rank, 0, communicator, &sendRequest[nbSendRequest]);
+                        nbSendRequest++;
+                }
+                if (nbRecvNode[rank] > 0)
+                {
+                        MPI_Irecv(recvBuffer[rank], recvMessageSize[rank], MPI_CHAR, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+                        nbRecvRequest++;
+                }
+        }
+        MPI_Waitall(nbRecvRequest, recvRequest, status);
+        MPI_Waitall(nbSendRequest, sendRequest, status);
+        char **sendBuffer2 = new char*[mpiSize];
+        char **recvBuffer2 = new char*[mpiSize];
+        double tic = cputime();
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                sentMessageSize[rank] = 0;
+                if (nbRecvNode[rank] > 0)
+                {
+                        Elt *recvElt = new Elt[nbRecvNode[rank]];
+                        pos[rank] = 0;
+                        for (int j = 0; j < nbRecvNode[rank]; j++)
+                        {
+                                unpackPolygon(recvElt[j], recvBuffer[rank], pos[rank]);
+                                cptEltGeom(recvElt[j], tgtGrid.pole);
+                                Node recvNode(recvElt[j].x, cptRadius(recvElt[j]), &recvElt[j]);
+                                recvNode.search(sstree.localTree.root);
+                                /* for a node holding an element of the target, loop throught candidates for intersecting source */
+                                for (list<NodePtr>::iterator it = (recvNode.intersectors).begin(); it != (recvNode.intersectors).end(); ++it)
+                                {
+                                        Elt *elt2 = (Elt *) ((*it)->data);
+                                        /* recvElt is target, elt2 is source */
+//                                      intersect(&recvElt[j], elt2);
+                                        intersect_ym(&recvElt[j], elt2);
+                                }
+                                if (recvElt[j].is.size() > 0) sentMessageSize[rank] += packIntersectionSize(recvElt[j]);
+                                // here recvNode goes out of scope
+                        }
+                        if (sentMessageSize[rank] > 0)
+                        {
+                                sentMessageSize[rank] += sizeof(int);
+                                sendBuffer2[rank] = new char[sentMessageSize[rank]];
+                                *((int *) sendBuffer2[rank]) = 0;
+                                pos[rank] = sizeof(int);
+                                for (int j = 0; j < nbRecvNode[rank]; j++)
+                                {
+                                        packIntersection(recvElt[j], sendBuffer2[rank], pos[rank]);
+                                        //FIXME should be deleted: recvElt[j].delete_intersections(); // intersection areas have been packed to buffer and won't be used any more
+                                }
+                        }
+                        delete [] recvElt;
+                }
+        }
+        delete [] pos;
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbSendNode[rank] > 0) delete [] sendBuffer[rank];
+                if (nbRecvNode[rank] > 0) delete [] recvBuffer[rank];
+                nbSendNode[rank] = 0;
+        }
+        if (verbose >= 2) cout << "Rank " << mpiRank << "  Compute (internal) intersection " << cputime() - tic << " s" << endl;
+        MPI_Alltoall(sentMessageSize, 1, MPI_INT, recvMessageSize, 1, MPI_INT, communicator);
+        for (int rank = 0; rank < mpiSize; rank++)
+                if (recvMessageSize[rank] > 0)
+                        recvBuffer2[rank] = new char[recvMessageSize[rank]];
+        nbSendRequest = 0;
+        nbRecvRequest = 0;
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (sentMessageSize[rank] > 0)
+                {
+                        MPI_Issend(sendBuffer2[rank], sentMessageSize[rank], MPI_CHAR, rank, 0, communicator, &sendRequest[nbSendRequest]);
+                        nbSendRequest++;
+                }
+                if (recvMessageSize[rank] > 0)
+                {
+                        MPI_Irecv(recvBuffer2[rank], recvMessageSize[rank], MPI_CHAR, rank, 0, communicator, &recvRequest[nbRecvRequest]);
+                        nbRecvRequest++;
+                }
+        }
+  MPI_Waitall(nbRecvRequest, recvRequest, status);
+  MPI_Waitall(nbSendRequest, sendRequest, status);
+        delete [] sendRequest;
+        delete [] recvRequest;
+        delete [] status;
+        for (int rank = 0; rank < mpiSize; rank++)
+        {
+                if (nbRecvNode[rank] > 0)
+                {
+                        if (sentMessageSize[rank] > 0)
+                                delete [] sendBuffer2[rank];
+                }
+                if (recvMessageSize[rank] > 0)
+                {
+                        unpackIntersection(elements, recvBuffer2[rank]);
+                        delete [] recvBuffer2[rank];
+                }
+        }
+        delete [] sendBuffer2;
+        delete [] recvBuffer2;
+        delete [] sendBuffer;
+        delete [] recvBuffer;
+        delete [] nbSendNode;
+        delete [] nbRecvNode;
+        delete [] sentMessageSize;
+        delete [] recvMessageSize;
+}
 Mapper::~Mapper()
+{
     delete [] remapMatrix;
     delete [] srcAddress;
     delete [] srcRank;
     delete [] dstAddress;
     if (neighbourElements) delete [] neighbourElements;
+}
+}
+        delete [] remapMatrix;
+        delete [] srcAddress;
+        delete [] srcRank;
+        delete [] dstAddress;
+        if (neighbourElements) delete [] neighbourElements;
+}
+}

XIOS/dev/branch_openmp/extern/remap/src/mapper.hpp

-                      r1134
+                      r1328
 #include "parallel_tree.hpp"
 #include "mpi.hpp"
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 namespace sphereRemap {
 …
+{
 public:
        Mapper(ep_lib::MPI_Comm comm=MPI_COMM_WORLD) : communicator(comm), verbose(SILENT), neighbourElements(NULL), sstree(comm) {}
+       Mapper(ep_lib::MPI_Comm comm) : communicator(comm), verbose(SILENT), neighbourElements(NULL), sstree(comm) {}
        ~Mapper();
        void setVerbosity(verbosity v) {verbose=v ;}

XIOS/dev/branch_openmp/extern/remap/src/mpi_cascade.cpp

r688	r1328
1	1	#include "mpi_cascade.hpp"
2	2	#include <iostream>
	3	using namespace ep_lib;
3	4
4	5	namespace sphereRemap {

XIOS/dev/branch_openmp/extern/remap/src/mpi_cascade.hpp

-                      r694
+                      r1328
+{
 public:
         CCascadeLevel(MPI_Comm comm) : comm(comm)
+        {
                 MPI_Comm_size(comm, &size);
                 MPI_Comm_rank(comm, &rank);
+        }
         int colour() const { return rank % group_size; };
         int key() const { return p_colour() + rank/(p_grp_size*group_size)*p_grp_size; }
+  CCascadeLevel(ep_lib::MPI_Comm comm) : comm(comm)
+  {
+    ep_lib::MPI_Comm_size(comm, &size);
+    ep_lib::MPI_Comm_rank(comm, &rank);
+  }
+  int colour() const { return rank % group_size; };
+  int key() const { return p_colour() + rank/(p_grp_size*group_size)*p_grp_size; }
         // perpendicular group
         int p_colour() const { return (rank%group_size + rank/group_size) % p_grp_size; }
         int p_key() const { return colour() + rank/(p_grp_size*group_size)*group_size; }
+  // perpendicular group
+  int p_colour() const { return (rank%group_size + rank/group_size) % p_grp_size; }
+  int p_key() const { return colour() + rank/(p_grp_size*group_size)*group_size; }
         MPI_Comm comm, pg_comm;
         int rank;
         int size;
         int group_size; // group_size and p_grp_size are interchanged?? FIXME
         int p_grp_size;
+  ep_lib::MPI_Comm comm, pg_comm;
+  int rank;
+  int size;
+  int group_size; // group_size and p_grp_size are interchanged?? FIXME
+  int p_grp_size;
 };
 …
+{
 public:
+        //
+        CMPICascade(int nodes_per_level, MPI_Comm comm);
+  CMPICascade(int nodes_per_level, ep_lib::MPI_Comm comm);
         int num_levels;
         std::vector<CCascadeLevel> level;
+  int num_levels;
+  std::vector<CCascadeLevel> level;
 };

XIOS/dev/branch_openmp/extern/remap/src/mpi_routing.cpp

-                      r1289
+                      r1328
 #include "timerRemap.hpp"
 #include <iostream>
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
+using namespace ep_lib;
 namespace sphereRemap {
 …
         CTimer::get("CMPIRouting::init(reduce_scatter)").print();
-        MPI_Info info_null;
-        // MPI_Alloc_mem(nbTarget *sizeof(int), info_null, &targetRank);
-        // MPI_Alloc_mem(nbSource *sizeof(int), info_null, &sourceRank);
         MPI_Alloc_mem(nbTarget *sizeof(int), MPI_INFO_NULL, &targetRank);
         MPI_Alloc_mem(nbSource *sizeof(int), MPI_INFO_NULL, &sourceRank);
 …
+        {
                 #ifdef _usingEP
                 MPI_Irecv(&sourceRank[i], 1, MPI_INT, -1, 0, communicator, &request[indexRequest]);
+                MPI_Irecv(&sourceRank[i], 1, MPI_INT, -2, 0, communicator, &request[indexRequest]);
                 #else
                 MPI_Irecv(&sourceRank[i], 1, MPI_INT, MPI_ANY_SOURCE, 0, communicator, &request[indexRequest]);
 …
+        {
                 #ifdef _usingEP
                 MPI_Irecv(&sourceRank[i], 1, MPI_INT, -1, 0, communicator, &request[indexRequest]);
+                MPI_Irecv(&sourceRank[i], 1, MPI_INT, -2, 0, communicator, &request[indexRequest]);
                 #else
                 MPI_Irecv(&sourceRank[i], 1, MPI_INT, MPI_ANY_SOURCE, 0, communicator, &request[indexRequest]);

XIOS/dev/branch_openmp/extern/remap/src/mpi_routing.hpp

-                      r694
+                      r1328
+{
         MPI_Comm communicator;
+        ep_lib::MPI_Comm communicator;
         int mpiRank;
         int mpiSize;
 …
 public:
         CMPIRouting(MPI_Comm comm);
+        CMPIRouting(ep_lib::MPI_Comm comm);
         ~CMPIRouting();
         template<typename T> void init(const std::vector<T>& route, CMPICascade *cascade = NULL);
 …
 template <typename T>
 void alltoalls_known(const std::vector<std::vector<T> >& send, std::vector<std::vector<T> >& recv,
                      const std::vector<int>& ranks, MPI_Comm communicator);
+                     const std::vector<int>& ranks, ep_lib::MPI_Comm communicator);
 template <typename T>
 void alltoalls_unknown(const std::vector<std::vector<T> >& send, std::vector<std::vector<T> >& recv,
                        const std::vector<int>& ranks, MPI_Comm communicator);
+                       const std::vector<int>& ranks, ep_lib::MPI_Comm communicator);
+}
 #endif

XIOS/dev/branch_openmp/extern/remap/src/node.cpp

-                      r1205
+                      r1328
 NodePtr insert(NodePtr thIs, NodePtr node)
+{
   int la = thIs->level; // node to be inserted
   int lb = node->level; // node where insertation
   assert(la < lb); // node to be inserted must have lower level then parent
   //if (thIs->parent) assert(find_in_tree1(thIs) == true);
   NodePtr q = NULL;
   NodePtr chd = NULL;
   node->move(thIs);
   if (la == lb - 1)
+  {
+        int la = thIs->level; // node to be inserted
+        int lb = node->level; // node where insertation
+        assert(la < lb); // node to be inserted must have lower level then parent
+        //if (thIs->parent) assert(find_in_tree1(thIs) == true);
+        NodePtr q = NULL;
+        NodePtr chd = NULL;
+        node->move(thIs);
+        if (la == lb - 1)
+        {
     node->child.push_back(thIs);
     thIs->parent = node;
     if (node->child.size() > MAX_NODE_SZ &&  node->tree->canSplit() ) // with us as additional child `node` is now too large :(
     return (node->reinserted || node->parent == NULL) ? split(node) : reinsert(node);
+  }
   else // la < lb - 1
+  {
     chd = thIs->closest(node->child);
     q = insert(thIs, chd);
+  }
   if ((node->updateCount + 1) % UPDATE_EVERY == 0)
     node->update();
   else
+  {
     if (q) node->remove(q);
     node->inflate(chd);
+  }
+                thIs->parent = node;
+                if (node->child.size() > MAX_NODE_SZ &&  node->tree->canSplit() ) // with us as additional child `node` is now too large :(
+                        return (node->reinserted || node->parent == NULL) ? split(node) : reinsert(node);
+        }
+        else // la < lb - 1
+        {
+                chd = thIs->closest(node->child);
+                q = insert(thIs, chd);
+        }
+        if ((node->updateCount + 1) % UPDATE_EVERY == 0)
+                node->update();
+        else
+        {
+                if (q) node->remove(q);
+                node->inflate(chd);
+        }
   return q;

XIOS/dev/branch_openmp/extern/remap/src/node.hpp

-                      r1153
+                      r1328
 struct Circle
+{
   Coord centre;
   double radius;
+        Coord centre;
+        double radius;
 };
 …
 struct Node
+{
   int level; /* FIXME leafs are 0 and root is max level? */
   int leafCount; /* number of leafs that are descendants of this node (the elements in it's cycle) */
   Coord centre;
   double radius;
   NodePtr parent, ref;
   std::vector<NodePtr> child;
   std::list<NodePtr> intersectors;
   bool reinserted;
   int updateCount;  // double var;
   CBasicTree* tree;
   void *data;
   int route;
+        int level; /* FIXME leafs are 0 and root is max level? */
+        int leafCount; /* number of leafs that are descendants of this node (the elements in it's cycle) */
+        Coord centre;
+        double radius;
+        NodePtr parent, ref;
+        std::vector<NodePtr> child;
+        std::list<NodePtr> intersectors;
+        bool reinserted;
+        int updateCount;  // double var;
+        CBasicTree* tree;
+        void *data;
+        int route;
   bool toDelete ;
   Node() : level(0), leafCount(1), centre(ORIGIN), radius(0), reinserted(false), updateCount(0), toDelete(false) {}
   Node(const Coord& centre, double radius, void *data)
     : level(0), leafCount(1), centre(centre), radius(radius), reinserted(false), updateCount(0), data(data), toDelete(false) {}
+        Node() : level(0), leafCount(1), centre(ORIGIN), radius(0), reinserted(false), updateCount(0), toDelete(false) {}
+        Node(const Coord& centre, double radius, void *data)
+                : level(0), leafCount(1), centre(centre), radius(radius), reinserted(false), updateCount(0), data(data), toDelete(false) {}
 //#ifdef DEBUG
 …
 //#endif
   void move(const NodePtr node);
   void remove(const NodePtr node);
   void inflate(const NodePtr node);
   void update();
+        void move(const NodePtr node);
+        void remove(const NodePtr node);
+        void inflate(const NodePtr node);
+        void update();
   void output(std::ostream& flux, int level, int color) ;
   NodePtr closest(std::vector<NodePtr>& list, int n = CLOSEST);
   NodePtr farthest(std::vector<NodePtr>& list);
   void findClosest(int level, NodePtr src, double& minDist, NodePtr &closest);
   void search(NodePtr node);
   bool centreInside(Node &node);
   bool intersects(NodePtr node);
   bool isInside(Node &node);
   int incluCheck();
+        NodePtr closest(std::vector<NodePtr>& list, int n = CLOSEST);
+        NodePtr farthest(std::vector<NodePtr>& list);
+        void findClosest(int level, NodePtr src, double& minDist, NodePtr &closest);
+        void search(NodePtr node);
+        bool centreInside(Node &node);
+        bool intersects(NodePtr node);
+        bool isInside(Node &node);
+        int incluCheck();
   void checkParent(void) ;
   void printChildren();
   void assignRoute(std::vector<int>::iterator& rank, int level);
   void assignCircleAndPropagateUp(Coord *centres, double *radia, int level);
   void printLevel(int level);
   void routeNode(NodePtr node, int level);
   void routingIntersecting(std::vector<Node>* routingList, NodePtr node);
   void routeIntersection(std::vector<int>& routes, NodePtr node);
+        void printChildren();
+        void assignRoute(std::vector<int>::iterator& rank, int level);
+        void assignCircleAndPropagateUp(Coord *centres, double *radia, int level);
+        void printLevel(int level);
+        void routeNode(NodePtr node, int level);
+        void routingIntersecting(std::vector<Node>* routingList, NodePtr node);
+        void routeIntersection(std::vector<int>& routes, NodePtr node);
   void getNodeLevel(int level,std::list<NodePtr>& NodeList) ;
   bool removeDeletedNodes(int assignLevel) ;

XIOS/dev/branch_openmp/extern/remap/src/parallel_tree.cpp

-                      r1295
+                      r1328
 #include "parallel_tree.hpp"
+using namespace ep_lib;
 namespace sphereRemap {
-extern CRemapGrid srcGrid;
-#pragma omp threadprivate(srcGrid)
-extern CRemapGrid tgtGrid;
-#pragma omp threadprivate(tgtGrid)
 static const int assignLevel = 2;
 …
+{
   int assignLevel = 2;
   int nbSampleNodes = 2*ipow(MAX_NODE_SZ + 1, assignLevel);
+        int assignLevel = 2;
+        int nbSampleNodes = 2*ipow(MAX_NODE_SZ + 1, assignLevel);
 …
   MPI_Comm_size(communicator,&commSize) ;
   // make multiple of two
   nbSampleNodes /= 2;
   nbSampleNodes *= 2;
   //assert( nbTot > nbSampleNodes*commSize) ;
+        // make multiple of two
+        nbSampleNodes /= 2;
+        nbSampleNodes *= 2;
+//  assert( nbTot > nbSampleNodes*commSize) ;
   int nbSampleNodes1 = nbSampleNodes * (nb1*commSize)/(1.*nbTot) ;
 …
   //assert(node.size() > nbSampleNodes);
   //assert(node2.size() > nbSampleNodes);
   //assert(node.size() + node2.size() > nbSampleNodes);
   vector<Node> sampleNodes; sampleNodes.reserve(nbSampleNodes1+nbSampleNodes2);
   vector<int> randomArray1(node.size());
   randomizeArray(randomArray1);
   vector<int> randomArray2(node2.size());
   randomizeArray(randomArray2);
   for (int i = 0; i <nbSampleNodes1; i++) sampleNodes.push_back(Node(node[randomArray1[i%nb1]].centre,  node[randomArray1[i%nb1]].radius, NULL));
   for (int i = 0; i <nbSampleNodes2; i++) sampleNodes.push_back(Node(node2[randomArray2[i%nb2]].centre, node2[randomArray2[i%nb2]].radius, NULL));
+//      assert(node.size() > nbSampleNodes);
+//      assert(node2.size() > nbSampleNodes);
+//      assert(node.size() + node2.size() > nbSampleNodes);
+        vector<Node> sampleNodes; sampleNodes.reserve(nbSampleNodes1+nbSampleNodes2);
+        vector<int> randomArray1(node.size());
+        randomizeArray(randomArray1);
+        vector<int> randomArray2(node2.size());
+        randomizeArray(randomArray2);
+        for (int i = 0; i <nbSampleNodes1; i++) sampleNodes.push_back(Node(node[randomArray1[i%nb1]].centre,  node[randomArray1[i%nb1]].radius, NULL));
+        for (int i = 0; i <nbSampleNodes2; i++) sampleNodes.push_back(Node(node2[randomArray2[i%nb2]].centre, node2[randomArray2[i%nb2]].radius, NULL));
         CTimer::get("buildParallelSampleTree").resume();
 …
         CTimer::get("parallelRouteNode").resume();
         vector<int> route(node.size());
+        cout<<"node.size = "<<node.size()<<endl;
         routeNodes(route /*out*/, node);
         CTimer::get("parallelRouteNode").suspend();

XIOS/dev/branch_openmp/extern/remap/src/parallel_tree.hpp

-                      r1134
+                      r1328
 #include "mpi_cascade.hpp"
 #include "mpi.hpp"
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 namespace sphereRemap {
 …
+{
 public:
         CParallelTree(ep_lib::MPI_Comm comm);
         ~CParallelTree();
+  CParallelTree(ep_lib::MPI_Comm comm);
+  ~CParallelTree();
         void build(vector<Node>& node, vector<Node>& node2);
+  void build(vector<Node>& node, vector<Node>& node2);
         void routeNodes(vector<int>& route, vector<Node>& nodes, int level = 0);
         void routeIntersections(vector<vector<int> >& route, vector<Node>& nodes, int level = 0);
+  void routeNodes(vector<int>& route, vector<Node>& nodes, int level = 0);
+  void routeIntersections(vector<vector<int> >& route, vector<Node>& nodes, int level = 0);
         int nbLocalElements;
         Elt* localElements;
+  int nbLocalElements;
+  Elt* localElements;
         CTree localTree;
+  CTree localTree;
 private:
         void updateCirclesForRouting(Coord rootCentre, double rootRadius, int level = 0);
         void buildSampleTreeCascade(vector<Node>& sampleNodes, int level = 0);
         void buildLocalTree(const vector<Node>& node, const vector<int>& route);
         void buildRouteTree();
+  void updateCirclesForRouting(Coord rootCentre, double rootRadius, int level = 0);
+  void buildSampleTreeCascade(vector<Node>& sampleNodes, int level = 0);
+  void buildLocalTree(const vector<Node>& node, const vector<int>& route);
+  void buildRouteTree();
         //CSampleTree sampleTree;
         vector<CSampleTree> treeCascade; // first for sample tree, then for routing tree
         CMPICascade cascade;
         ep_lib::MPI_Comm communicator ;
+  //CSampleTree sampleTree;
+  vector<CSampleTree> treeCascade; // first for sample tree, then for routing tree
+  CMPICascade cascade;
+  ep_lib::MPI_Comm communicator ;
 };

XIOS/dev/branch_openmp/extern/remap/src/polyg.cpp

-                      r1289
+                      r1328
 #include "polyg.hpp"
-#include <stdio.h>
 namespace sphereRemap {
 …
 Coord barycentre(const Coord *x, int n)
+{
         if (n == 0) return ORIGIN;
         Coord bc = ORIGIN;
         for (int i = 0; i < n; i++)
+        {
                 bc = bc + x[i];
+        }
         /* both distances can be equal down to roundoff when norm(bc) < mashineepsilon
            which can occur when weighted with tiny area */
+  //assert(squaredist(bc, proj(bc)) <= squaredist(bc, proj(bc * (-1.0))));
+        assert(squaredist(bc, proj(bc)) <= squaredist(bc, proj(bc * (-1.0))));
+        //if (squaredist(bc, proj(bc)) > squaredist(bc, proj(bc * (-1.0)))) return proj(bc * (-1.0));
         return proj(bc);
 …
+{
         if (N < 3)
                 return 0; /* polygons with less than three vertices have zero area */
+          return 0; /* polygons with less than three vertices have zero area */
         Coord t[3];
         t[0] = barycentre(x, N);
 …
                 t[1] = x[i];
                 t[2] = x[ii];
     double sc=scalarprod(crossprod(t[1] - t[0], t[2] - t[0]), t[0]) ;
+                double sc=scalarprod(crossprod(t[1] - t[0], t[2] - t[0]), t[0]) ;
                 assert(sc >= -1e-10); // Error: tri a l'env (wrong orientation)
                 double area_gc = triarea(t[0], t[1], t[2]);
-                //if(area_gc<=0) printf("area_gc = %e\n", area_gc);
                 double area_sc_gc_moon = 0;
                 if (d[i]) /* handle small circle case */
 …
                         char sgl = (mext > 0) ? -1 : 1;
                         area_sc_gc_moon = sgl * alun(arcdist(t[1], t[2]), fabs(scalarprod(t[1], pole)));
-                        //if(area_sc_gc_moon<=0) printf("area_sc_gc_moon = %e\n", area_sc_gc_moon);
                         gg_exact = gg_exact + sc_gc_moon_normalintegral(t[1], t[2], pole);
+                }
                 area += area_gc + area_sc_gc_moon; /* for "spherical circle segment" sum triangular part (at) and "small moon" and => account for small circle */
                 g[i] = barycentre(t, 3) * (area_gc + area_sc_gc_moon);
-                //printf("g[%d] = (%e,%e,%e) * (%e+%e) = (%e,%e,%e) norm = %e\n", i, barycentre(t, 3).x, barycentre(t, 3).y, barycentre(t, 3).z, area_gc,  area_sc_gc_moon, g[i].x, g[i].y, g[i].z, norm(g[i]));
+        }
         gg = barycentre(g, N);

XIOS/dev/branch_openmp/extern/remap/src/timerRemap.cpp

-                      r1146
+                      r1328
 #include <map>
 #include <iostream>
+using namespace ep_lib;
 namespace sphereRemap {
 …
 using namespace std;
+map<string,CTimer*> *CTimer::allTimer = 0;
+//map<string,CTimer*> CTimer::allTimer;
+map<string,CTimer*> *CTimer::allTimer_ptr = 0;
 CTimer::CTimer(const string& name_) : name(name_)
 …
 CTimer& CTimer::get(const string name)
+{
-        if(allTimer == 0) allTimer = new map<string,CTimer*>;
         map<string,CTimer*>::iterator it;
+        it=(*allTimer).find(name);
+        if (it==(*allTimer).end()) it=(*allTimer).insert(pair<string,CTimer*>(name,new CTimer(name))).first;
+        if(allTimer_ptr == 0) allTimer_ptr = new map<string,CTimer*>;
+        //it=allTimer.find(name);
+        it=allTimer_ptr->find(name);
+        //if (it==allTimer.end()) it=allTimer.insert(pair<string,CTimer*>(name,new CTimer(name))).first;
+        if (it==allTimer_ptr->end()) it=allTimer_ptr->insert(pair<string,CTimer*>(name,new CTimer(name))).first;
         return *(it->second);
+}

XIOS/dev/branch_openmp/extern/remap/src/timerRemap.hpp

-                      r1146
+                      r1328
     void print(void);
     //static map<string,CTimer*> allTimer;
+    static map<string,CTimer*> *allTimer;
+    #pragma omp threadprivate(allTimer)
+    static map<string,CTimer*> *allTimer_ptr;
     static double getTime(void);
     static CTimer& get(string name);

XIOS/dev/branch_openmp/extern/remap/src/tree.cpp

-                      r1172
+                      r1328
 void CBasicTree::routeNodes(vector<int>& route, vector<Node>& nodes, int assignLevel)
+{
   for (int i = 0; i < nodes.size(); i++)
+  {
     root->routeNode(&nodes[i], assignLevel);
     route[i] = nodes[i].route;
+  }
+        for (int i = 0; i < nodes.size(); i++)
+        {
+                root->routeNode(&nodes[i], assignLevel);
+                route[i] = nodes[i].route;
+        }
+}
 void CBasicTree::routeIntersections(vector<vector<int> >& routes, vector<Node>& nodes)
+{
   for (int i = 0; i < nodes.size(); i++)
     root->routeIntersection(routes[i], &nodes[i]);
+        for (int i = 0; i < nodes.size(); i++)
+                root->routeIntersection(routes[i], &nodes[i]);
+}
 void CBasicTree::build(vector<Node>& nodes)
+{
   newRoot(1);
   insertNodes(nodes);
+        newRoot(1);
+        insertNodes(nodes);
+}
 void CBasicTree::output(ostream& flux, int level)
+{
   root->output(flux,level,0) ;
+        root->output(flux,level,0) ;
+}
 void CBasicTree::slim(int nbIts)
+{
   for (int i = 0; i < nbIts; i++)
+  {
     for (int level = root->level - 1; level > 0; level--)
+    {
       slim2(root, level);
       ri = 0;
       emptyPool();
+    }
     for (int level = 2; level < root->level; level++)
+    {
       slim2(root, level);
       ri = 0;
       emptyPool();
+    }
+  }
+        for (int i = 0; i < nbIts; i++)
+        {
+                for (int level = root->level - 1; level > 0; level--)
+                {
+                        slim2(root, level);
+                        ri = 0;
+                        emptyPool();
+                }
+                for (int level = 2; level < root->level; level++)
+                {
+                        slim2(root, level);
+                        ri = 0;
+                        emptyPool();
+                }
+        }
+}
 …
 void CBasicTree::insertNode(NodePtr node)
+{
   node->tree = this;
   increaseLevelSize(0);
   push_back(node);
   NodePtr q;
   while (pool.size())
+  {
     q = pool.front();
     pool.pop_front();
     q = insert(q, root);
     if (ri)
+    {
       delete q;
       ri = 0;
+    }
+  }
+        node->tree = this;
+        increaseLevelSize(0);
+        push_back(node);
+        NodePtr q;
+        while (pool.size())
+        {
+                q = pool.front();
+                pool.pop_front();
+                q = insert(q, root);
+                if (ri)
+                {
+                        delete q;
+                        ri = 0;
+                }
+        }
+}
 void CBasicTree::emptyPool(void)
+{
   while (pool.size())
+  {
     NodePtr q = pool.front();
     pool.pop_front();
     q = insert(q, root);
     if (ri)
+    {
       delete q;
       ri = 0;
+    }
+  }
+        while (pool.size())
+        {
+                NodePtr q = pool.front();
+                pool.pop_front();
+                q = insert(q, root);
+                if (ri)
+                {
+                        delete q;
+                        ri = 0;
+                }
+        }
+}
 …
         root->parent = 0;
         root->leafCount = 0;
+        // initialize root node on the sphere
+        root->centre.x=1 ;
+        root->centre.y=0 ;
+        root->centre.z=0 ;
+// initialize root node on the sphere
+  root->centre.x=1 ; root->centre.y=0 ; root->centre.z=0 ;
         root->radius = 0.;
         root->reinserted = false;

XIOS/dev/branch_openmp/extern/remap/src/tree.hpp

-                      r1172
+                      r1328
 class CBasicTree
+{
   public:
+public:
   NodePtr root; /* The main tree is stored as Nodes which can be reached through traversal starting here */
   NodePtr ref; // FIXME this reference, set by a node is odd, try to remove
   int ri; /** this is set to one by a node in case of reinsertion */
   vector<int> levelSize; /** e.g. levelSize[0] == leafs.size() */
   vector<Node> leafs; /** leafs are stored in vector for easy access and rest of the tree nodes as separate allocations, only reachable through tree traversal */
+        NodePtr root; /* The main tree is stored as Nodes which can be reached through traversal starting here */
+        NodePtr ref; // FIXME this reference, set by a node is odd, try to remove
+        int ri; /** this is set to one by a node in case of reinsertion */
+        vector<int> levelSize; /** e.g. levelSize[0] == leafs.size() */
+        vector<Node> leafs; /** leafs are stored in vector for easy access and rest of the tree nodes as separate allocations, only reachable through tree traversal */
   CBasicTree() : ri(0), levelSize(MAX_LEVEL_SIZE), root(NULL), isAssignedLevel(false), okSplit(true), isActiveOkSplit(false) {}
   ~CBasicTree();
   void build(vector<Node>& nodes);
   void slim(int nbIts = 1);
   virtual void insertNodes(vector<Node>& node) = 0;
+        CBasicTree() : ri(0), levelSize(MAX_LEVEL_SIZE), root(NULL), isAssignedLevel(false), okSplit(true), isActiveOkSplit(false) {}
+        ~CBasicTree();
+        void build(vector<Node>& nodes);
+        void slim(int nbIts = 1);
+        virtual void insertNodes(vector<Node>& node) = 0;
   void routeNodes(vector<int>& route, vector<Node>& nodes, int assignLevel);
   void routeIntersections(vector<vector<int> >& route, vector<Node>& nodes);
+        void routeNodes(vector<int>& route, vector<Node>& nodes, int assignLevel);
+        void routeIntersections(vector<vector<int> >& route, vector<Node>& nodes);
   void push_back(NodePtr node);
   void push_front(NodePtr node);
   void increaseLevelSize(int level);
   void decreaseLevelSize(int level);
   void newRoot(int level);
   void insertNode(NodePtr node);
+        void push_back(NodePtr node);
+        void push_front(NodePtr node);
+        void increaseLevelSize(int level);
+        void decreaseLevelSize(int level);
+        void newRoot(int level);
+        void insertNode(NodePtr node);
   void output(ostream& flux, int level) ;
   int keepNodes;
+        int keepNodes;
   bool isAssignedLevel ;
   int assignLevel;
 …
   private:
   deque<NodePtr > pool;
+private:
+        deque<NodePtr > pool;
   bool okSplit ;
   protected:
+protected:
   void emptyPool();
+  CBasicTree(int keepNodes_, int assignLevel_) : ri(0), levelSize(MAX_LEVEL_SIZE), root(NULL), keepNodes(keepNodes_), assignLevel(assignLevel_), isAssignedLevel(true),
+                                                 okSplit(true), isActiveOkSplit(false) {}
+  CBasicTree(int keepNodes_, int assignLevel_) : ri(0), levelSize(MAX_LEVEL_SIZE), root(NULL), keepNodes(keepNodes_), assignLevel(assignLevel_), isAssignedLevel(true), okSplit(true), isActiveOkSplit(false) {}
 };
 class CTree : public CBasicTree
+{
   public:
   void insertNodes(vector<Node>& nodes);
+public:
+        void insertNodes(vector<Node>& nodes);
 };
 …
+{
   public:
   CSampleTree(int keepNodes_, int assignLevel_) : CBasicTree(keepNodes_,assignLevel_) {}
+public:
+        CSampleTree(int keepNodes_, int assignLevel_) : CBasicTree(keepNodes_,assignLevel_) {}
   void slimAssignedLevel() ;
   void removeExtraNode(void) ;
   void insertNodes(vector<Node>& nodes);
+        void insertNodes(vector<Node>& nodes);
 };

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_fortran.cpp

r1287	r1328
31	31	{
32	32	fc_comm_map.insert(std::make_pair( std::make_pair( fint, omp_get_thread_num()) , comm));
33		//printf("EP_Comm_c2f : MAP %p insert: %d, %d, %p\n", &fc_comm_map, fint, omp_get_thread_num(), comm.ep_comm_ptr);
	33	printf("EP_Comm_c2f : MAP %p insert: %d, %d, %p\n", &fc_comm_map, fint, omp_get_thread_num(), comm.ep_comm_ptr);
34	34	}
35	35	}
…	…
54	54	MPI_Comm comm_ptr;
55	55	comm_ptr = it->second;
56		//printf("EP_Comm_f2c : MAP %p find: %d, %d, %p\n", &fc_comm_map, it->first.first, it->first.second, comm_ptr.ep_comm_ptr);
	56	printf("EP_Comm_f2c : MAP %p find: %d, %d, %p\n", &fc_comm_map, it->first.first, it->first.second, comm_ptr.ep_comm_ptr);
57	57	return comm_ptr;
58	58	}

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_intercomm.cpp

-                      r1287
+                      r1328
       MPI_Waitall(2, request, status);
+      //MPI_Send(&leader_ranks[0], 3, static_cast< ::MPI_Datatype>(MPI_INT), remote_leader, tag, peer_comm);
+      //MPI_Recv(&leader_ranks[3], 3, static_cast< ::MPI_Datatype>(MPI_INT), remote_leader, tag, peer_comm, &status[1]);
+    }

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_lib.cpp

-                      r1295
+                      r1328
 #pragma omp threadprivate(EP_PendingRequests)
 namespace ep_lib
+{
+  bool MPI_Comm::is_null()
+  {
+    if(!this->is_intercomm)
+      return this->mpi_comm == MPI_COMM_NULL.mpi_comm;
+    else
+      return this->ep_comm_ptr->intercomm->mpi_inter_comm == MPI_COMM_NULL.mpi_comm;
+  }
   int tag_combine(int real_tag, int src, int dest)

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_send.cpp

-                      r1295
+                      r1328
     if(!comm.is_ep)
       return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm.mpi_comm));
+    MPI_Request request;
+    MPI_Status status;
+    MPI_Isend(buf, count, datatype, dest, tag, comm, &request);
+    MPI_Wait(&request, &status);
+    if(comm.is_intercomm)
+    {
+      MPI_Request request;
+      MPI_Status status;
+      MPI_Isend(buf, count, datatype, dest, tag, comm, &request);
+      MPI_Wait(&request, &status);
+    }
+    else
+    {
+      int ep_src_loc  = comm.ep_comm_ptr->size_rank_info[1].first;
+      int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first;
+      int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc);
+      int mpi_dest    = comm.ep_comm_ptr->comm_list->rank_map->at(dest).second;
+      ::MPI_Send(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm.mpi_comm));
+      //printf("call mpi_send for intracomm, dest = %d, tag = %d\n", dest, tag);
+    }
     //check_sum_send(buf, count, datatype, dest, tag, comm);

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_type.hpp

r1295	r1328
344	344	}
345	345
	346	bool is_null();
	347
346	348	};
347	349

XIOS/dev/branch_openmp/inputs/REMAP/iodef.xml

-                      r1220
+                      r1328
       <file_group id="write_files" >
         <file id="output_2D" name="output_2D" enabled=".TRUE.">
+        <file id="output_2D" name="output_2D" output_freq="1ts" enabled=".TRUE.">
           <field field_ref="src_field_2D" name="field_src" enabled=".TRUE."/>
           <field field_ref="src_field_2D_clone" name="field_src_clone" default_value="100000" enabled=".TRUE."/>
 …
           <field field_ref="dst_field_2D" name="field_dst_regular_1" enabled=".TRUE." />
           <field field_ref="dst_field_2D_regular_pole" name="field_dst_regular_2" enabled=".TRUE."/>
           <field field_ref="dst_field_2D_clone" name="field_dst_regular_3" detect_missing_value=".false." default_value="100000" enabled=".TRUE."/>
+          <field field_ref="dst_field_2D_clone" name="field_dst_regular_3" detect_missing_value=".false." default_value="100000" enabled=".FALSE."/>
           <field field_ref="dst_field_2D_extract" name="field_dst_regular_4" enabled=".TRUE."/>
         </file>

XIOS/dev/branch_openmp/src/array_new.hpp

r1134	r1328
554	554	TinyVector<int,N_rank> vect;
555	555	size_t ne;
556
	556
557	557	ret = buffer.get(numDim);
558	558	ret &= buffer.get(vect.data(), N_rank);

XIOS/dev/branch_openmp/src/attribute_enum.hpp

-                      r1134
+                      r1328
 namespace xios
+{
     /// ////////////////////// DÃ©clarations ////////////////////// ///
     /*!
       \class CAttributeEnum
       This class implements the attribute representing enumeration
+      /// ////////////////////// Déclarations ////////////////////// ///
+        /*!
+        \class CAttributeEnum
+        This class implements the attribute representing enumeration
       */
     template <class T>
         class CAttributeEnum : public CAttribute, public CEnum<T>
+    {
+      template <class T>
+         class CAttributeEnum : public CAttribute, public CEnum<T>
+      {
         typedef typename T::t_enum T_enum ;
         public :
         /// Constructeurs ///
         explicit CAttributeEnum(const StdString & id);
         CAttributeEnum(const StdString & id,
                 xios_map<StdString, CAttribute*> & umap);
         CAttributeEnum(const StdString & id, const T_enum & value);
         CAttributeEnum(const StdString & id, const T_enum & value,
                 xios_map<StdString, CAttribute*> & umap);
+            /// Constructeurs ///
+            explicit CAttributeEnum(const StdString & id);
+            CAttributeEnum(const StdString & id,
+                               xios_map<StdString, CAttribute*> & umap);
+            CAttributeEnum(const StdString & id, const T_enum & value);
+            CAttributeEnum(const StdString & id, const T_enum & value,
+                               xios_map<StdString, CAttribute*> & umap);
         /// Accesseur ///
         T_enum getValue(void) const;
         string getStringValue(void) const;
+            /// Accesseur ///
+            T_enum getValue(void) const;
+            string getStringValue(void) const;
+        /// Mutateurs ///
+        void setValue(const T_enum & value);
+            /// Mutateurs ///
+            void setValue(const T_enum & value);
+            void set(const CAttribute& attr) ;
+            void set(const CAttributeEnum& attr) ;
+            void reset(void);
+            void setInheritedValue(const CAttributeEnum& attr );
+            void setInheritedValue(const CAttribute& attr );
+            T_enum getInheritedValue(void)  const;
+            string getInheritedStringValue(void) const;
+            bool hasInheritedValue(void) const;
+            bool isEqual(const CAttributeEnum& attr );
+            bool isEqual(const CAttribute& attr );
+        void set(const CAttribute& attr) ;
+        void set(const CAttributeEnum& attr) ;
+        void reset(void);
+            /// Destructeur ///
+            virtual ~CAttributeEnum(void) { }
+        void setInheritedValue(const CAttributeEnum& attr );
+        void setInheritedValue(const CAttribute& attr );
+        T_enum getInheritedValue(void)  const;
+        string getInheritedStringValue(void) const;
+        bool hasInheritedValue(void) const;
+            /// Operateur ///
+            CAttributeEnum& operator=(const T_enum & value);
+        bool isEqual(const CAttributeEnum& attr );
+        bool isEqual(const CAttribute& attr );
+            /// Autre ///
+            virtual StdString toString(void) const { return _toString();}
+            virtual void fromString(const StdString & str) { if (str==resetInheritanceStr) { reset(); _canInherite=false ;}  else _fromString(str);}
+        /// Destructeur ///
+        virtual ~CAttributeEnum(void) { }
+            virtual bool toBuffer  (CBufferOut& buffer) const { return _toBuffer(buffer);}
+            virtual bool fromBuffer(CBufferIn& buffer) { return _fromBuffer(buffer); }
+            virtual void generateCInterface(ostream& oss,const string& className) ;
+            virtual void generateFortran2003Interface(ostream& oss,const string& className) ;
+            virtual void generateFortranInterfaceDeclaration_(ostream& oss,const string& className) ;
+            virtual void generateFortranInterfaceBody_(ostream& oss,const string& className) ;
+            virtual void generateFortranInterfaceDeclaration(ostream& oss,const string& className) ;
+            virtual void generateFortranInterfaceGetDeclaration_(ostream& oss,const string& className) ;
+            virtual void generateFortranInterfaceGetBody_(ostream& oss,const string& className) ;
+            virtual void generateFortranInterfaceGetDeclaration(ostream& oss,const string& className) ;
+        /// Operateur ///
+        CAttributeEnum& operator=(const T_enum & value);
+        /// Autre ///
+        virtual StdString toString(void) const { return _toString();}
+        virtual void fromString(const StdString & str) { if (str==resetInheritanceStr) { reset(); _canInherite=false ;}  else _fromString(str);}
+        virtual bool toBuffer  (CBufferOut& buffer) const { return _toBuffer(buffer);}
+        virtual bool fromBuffer(CBufferIn& buffer) { return _fromBuffer(buffer); }
+        virtual void generateCInterface(ostream& oss,const string& className) ;
+        virtual void generateFortran2003Interface(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceDeclaration_(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceBody_(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceDeclaration(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceGetDeclaration_(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceGetBody_(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceGetDeclaration(ostream& oss,const string& className) ;
+        private :
+        StdString _toString(void) const;
+        void _fromString(const StdString & str);
+        bool _toBuffer  (CBufferOut& buffer) const;
+        bool _fromBuffer(CBufferIn& buffer) ;
+        CEnum<T> inheritedValue ;
+    }; // class CAttributeEnum
+         private :
+          StdString _toString(void) const;
+          void _fromString(const StdString & str);
+          bool _toBuffer  (CBufferOut& buffer) const;
+          bool _fromBuffer(CBufferIn& buffer) ;
+          CEnum<T> inheritedValue ;
+      }; // class CAttributeEnum
 } // namespace xios
 #endif // __XIOS_ATTRIBUTE_ENUM__

XIOS/dev/branch_openmp/src/attribute_enum_impl.hpp

-                      r1134
+                      r1328
 namespace xios
+{
   /// ////////////////////// DÃ©finitions ////////////////////// ///
+  /// ////////////////////// Définitions ////////////////////// ///
   template <class T>
   CAttributeEnum<T>::CAttributeEnum(const StdString & id)
 …
      umap.insert(umap.end(), std::make_pair(id, this));
+  }
   template <class T>
   CAttributeEnum<T>::CAttributeEnum
 …
      umap.insert(umap.end(), std::make_pair(id, this));
+  }
   ///--------------------------------------------------------------
   template <class T>
 …
      return CEnum<T>::get();
+  }
   template <class T>
   string CAttributeEnum<T>::getStringValue(void) const
+  {
+    return CEnum<T>::toString();
+  }
+     return CEnum<T>::toString();
+  }
   template <class T>
 …
   void CAttributeEnum<T>::set(const CAttribute& attr)
+  {
      this->set(dynamic_cast<const CAttributeEnum<T>& >(attr));
+  }
   template <class T>
+    this->set(dynamic_cast<const CAttributeEnum<T>& >(attr));
+  }
+ template <class T>
   void CAttributeEnum<T>::set(const CAttributeEnum& attr)
+  {
      CEnum<T>::set(attr);
+  }
+    CEnum<T>::set(attr);
+  }
   template <class T>
   void CAttributeEnum<T>::setInheritedValue(const CAttribute& attr)
+  {
      this->setInheritedValue(dynamic_cast<const CAttributeEnum<T>& >(attr));
+  }
+    this->setInheritedValue(dynamic_cast<const CAttributeEnum<T>& >(attr));
+  }
   template <class T>
   void CAttributeEnum<T>::setInheritedValue(const CAttributeEnum& attr)
+  {
      if (this->isEmpty() && _canInherite && attr.hasInheritedValue()) inheritedValue.set(attr.getInheritedValue());
+  }
+    if (this->isEmpty() && _canInherite && attr.hasInheritedValue()) inheritedValue.set(attr.getInheritedValue());
+  }
   template <class T>
   typename T::t_enum CAttributeEnum<T>::getInheritedValue(void) const
+  {
      if (this->isEmpty()) return inheritedValue.get();
      else return getValue();
+  }
   template <class T>
       string CAttributeEnum<T>::getInheritedStringValue(void) const
+      {
           if (this->isEmpty()) return inheritedValue.toString();
           else return CEnum<T>::toString();;
+      }
   template <class T>
       bool CAttributeEnum<T>::hasInheritedValue(void) const
+      {
           return !this->isEmpty() || !inheritedValue.isEmpty();
+      }
   template <class T>
       bool CAttributeEnum<T>::isEqual(const CAttribute& attr)
+      {
           return (this->isEqual(dynamic_cast<const CAttributeEnum<T>& >(attr)));
+      }
   template <class T>
       bool CAttributeEnum<T>::isEqual(const CAttributeEnum& attr)
+      {
           return ((dynamic_cast<const CEnum<T>& >(*this)) == (dynamic_cast<const CEnum<T>& >(attr)));
+      }
+    if (this->isEmpty()) return inheritedValue.get();
+    else return getValue();
+  }
+  template <class T>
+  string CAttributeEnum<T>::getInheritedStringValue(void) const
+  {
+     if (this->isEmpty()) return inheritedValue.toString();
+     else return CEnum<T>::toString();;
+  }
+  template <class T>
+  bool CAttributeEnum<T>::hasInheritedValue(void) const
+  {
+    return !this->isEmpty() || !inheritedValue.isEmpty();
+  }
+  template <class T>
+  bool CAttributeEnum<T>::isEqual(const CAttribute& attr)
+  {
+    return (this->isEqual(dynamic_cast<const CAttributeEnum<T>& >(attr)));
+  }
+  template <class T>
+  bool CAttributeEnum<T>::isEqual(const CAttributeEnum& attr)
+  {
+    return ((dynamic_cast<const CEnum<T>& >(*this)) == (dynamic_cast<const CEnum<T>& >(attr)));
+  }
   //---------------------------------------------------------------
   template <class T>
       CAttributeEnum<T>& CAttributeEnum<T>::operator=(const T_enum & value)
+      {
           this->setValue(value);
           return *this;
+      }
+  CAttributeEnum<T>& CAttributeEnum<T>::operator=(const T_enum & value)
+  {
+     this->setValue(value);
+     return *this;
+  }
   //---------------------------------------------------------------
   template <class T>
       StdString CAttributeEnum<T>::_toString(void) const
+      {
           StdOStringStream oss;
           if (!CEnum<T>::isEmpty() && this->hasId())
               oss << this->getName() << "=\"" << CEnum<T>::toString() << "\"";
           return (oss.str());
+      }
   template <class T>
       void CAttributeEnum<T>::_fromString(const StdString & str)
+      {
           CEnum<T>::fromString(str);
+      }
   template <class T>
       bool CAttributeEnum<T>::_toBuffer (CBufferOut& buffer) const
+      {
           return CEnum<T>::toBuffer(buffer);
+      }
   template <class T>
       bool CAttributeEnum<T>::_fromBuffer(CBufferIn& buffer)
+      {
           return CEnum<T>::fromBuffer(buffer);
+      }
   template <typename T>
       void CAttributeEnum<T>::generateCInterface(ostream& oss,const string& className)
+      {
           CInterface::AttributeCInterface<CEnumBase>(oss, className, this->getName());
+      }
   template <typename T>
       void CAttributeEnum<T>::generateFortran2003Interface(ostream& oss,const string& className)
+      {
           CInterface::AttributeFortran2003Interface<string>(oss, className, this->getName());
+      }
   template <typename T>
       void CAttributeEnum<T>::generateFortranInterfaceDeclaration_(ostream& oss,const string& className)
+      {
           CInterface::AttributeFortranInterfaceDeclaration<string>(oss, className, this->getName()+"_");
+      }
   template <typename T>
       void CAttributeEnum<T>::generateFortranInterfaceBody_(ostream& oss,const string& className)
+      {
           CInterface::AttributeFortranInterfaceBody<string>(oss, className, this->getName());
+      }
   template <typename T>
       void CAttributeEnum<T>::generateFortranInterfaceDeclaration(ostream& oss,const string& className)
+      {
           CInterface::AttributeFortranInterfaceDeclaration<string>(oss, className, this->getName());
+      }
   template <typename T>
       void CAttributeEnum<T>::generateFortranInterfaceGetDeclaration_(ostream& oss,const string& className)
+      {
           CInterface::AttributeFortranInterfaceGetDeclaration<string>(oss, className, this->getName()+"_");
+      }
   template <typename T>
       void CAttributeEnum<T>::generateFortranInterfaceGetBody_(ostream& oss,const string& className)
+      {
           CInterface::AttributeFortranInterfaceGetBody<string>(oss, className, this->getName());
+      }
   template <typename T>
       void CAttributeEnum<T>::generateFortranInterfaceGetDeclaration(ostream& oss,const string& className)
+      {
           CInterface::AttributeFortranInterfaceGetDeclaration<string>(oss, className, this->getName());
+      }
+  StdString CAttributeEnum<T>::_toString(void) const
+  {
+     StdOStringStream oss;
+     if (!CEnum<T>::isEmpty() && this->hasId())
+        oss << this->getName() << "=\"" << CEnum<T>::toString() << "\"";
+     return (oss.str());
+  }
+  template <class T>
+  void CAttributeEnum<T>::_fromString(const StdString & str)
+  {
+    CEnum<T>::fromString(str);
+  }
+  template <class T>
+  bool CAttributeEnum<T>::_toBuffer (CBufferOut& buffer) const
+  {
+     return CEnum<T>::toBuffer(buffer);
+  }
+  template <class T>
+  bool CAttributeEnum<T>::_fromBuffer(CBufferIn& buffer)
+  {
+    return CEnum<T>::fromBuffer(buffer);
+  }
+  template <typename T>
+  void CAttributeEnum<T>::generateCInterface(ostream& oss,const string& className)
+  {
+    CInterface::AttributeCInterface<CEnumBase>(oss, className, this->getName());
+  }
+  template <typename T>
+  void CAttributeEnum<T>::generateFortran2003Interface(ostream& oss,const string& className)
+  {
+    CInterface::AttributeFortran2003Interface<string>(oss, className, this->getName());
+  }
+  template <typename T>
+  void CAttributeEnum<T>::generateFortranInterfaceDeclaration_(ostream& oss,const string& className)
+  {
+    CInterface::AttributeFortranInterfaceDeclaration<string>(oss, className, this->getName()+"_");
+  }
+  template <typename T>
+  void CAttributeEnum<T>::generateFortranInterfaceBody_(ostream& oss,const string& className)
+  {
+    CInterface::AttributeFortranInterfaceBody<string>(oss, className, this->getName());
+  }
+  template <typename T>
+  void CAttributeEnum<T>::generateFortranInterfaceDeclaration(ostream& oss,const string& className)
+  {
+    CInterface::AttributeFortranInterfaceDeclaration<string>(oss, className, this->getName());
+  }
+  template <typename T>
+  void CAttributeEnum<T>::generateFortranInterfaceGetDeclaration_(ostream& oss,const string& className)
+  {
+    CInterface::AttributeFortranInterfaceGetDeclaration<string>(oss, className, this->getName()+"_");
+  }
+  template <typename T>
+  void CAttributeEnum<T>::generateFortranInterfaceGetBody_(ostream& oss,const string& className)
+  {
+    CInterface::AttributeFortranInterfaceGetBody<string>(oss, className, this->getName());
+  }
+  template <typename T>
+  void CAttributeEnum<T>::generateFortranInterfaceGetDeclaration(ostream& oss,const string& className)
+  {
+    CInterface::AttributeFortranInterfaceGetDeclaration<string>(oss, className, this->getName());
+  }
 } // namespace xios
 #endif // __XIOS_ATTRIBUTE_ENUM_IMPL_HPP__

XIOS/dev/branch_openmp/src/attribute_map.hpp

r1134	r1328
76	76	/// PropriÃ©tÃ© statique ///
77	77	static CAttributeMap * Current;
78		~~#pragma omp threadprivate (Current)~~
79	78
80	79	}; // class CAttributeMap

XIOS/dev/branch_openmp/src/buffer_client.cpp

-                      r1205
+                      r1328
 #include "mpi.hpp"
 #include "tracer.hpp"
+using namespace ep_lib;
 namespace xios
 …
     buffer[1] = new char[bufferSize];
     retBuffer = new CBufferOut(buffer[current], bufferSize);
-    #pragma omp critical (_output)
     info(10) << "CClientBuffer: allocated 2 x " << bufferSize << " bytes for server " << serverRank << " with a maximum of " << maxBufferedEvents << " buffered events" << endl;
+  }

XIOS/dev/branch_openmp/src/buffer_client.hpp

-                      r1205
+                      r1328
 #include "mpi.hpp"
 #include "cxios.hpp"
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 namespace xios
 …
     public:
       static size_t maxRequestSize;
-      #pragma omp threadprivate(maxRequestSize)
       CClientBuffer(MPI_Comm intercomm, int serverRank, StdSize bufferSize, StdSize estimatedMaxEventSize, StdSize maxBufferedEvents);
+      CClientBuffer(ep_lib::MPI_Comm intercomm, int serverRank, StdSize bufferSize, StdSize estimatedMaxEventSize, StdSize maxBufferedEvents);
       ~CClientBuffer();
 …
       bool pending;
       MPI_Request request;
+      ep_lib::MPI_Request request;
       CBufferOut* retBuffer;
       const MPI_Comm interComm;
+      const ep_lib::MPI_Comm interComm;
   };
+}

XIOS/dev/branch_openmp/src/buffer_server.hpp

r1134	r1328
4	4	#include "xios_spl.hpp"
5	5	#include "buffer.hpp"
6		#include "mpi~~_std~~.hpp"
	6	#include "mpi.hpp"
7	7	#include "cxios.hpp"
8	8

XIOS/dev/branch_openmp/src/calendar.cpp

r1134	r1328
117	117	const CDate& CCalendar::update(int step)
118	118	{
119		#pragma omp critical (_output)
120		info(80)<< "update step : " << step << " timestep " << this->timestep << std::endl;
	119	info(20) << "update step : " << step << " timestep " << this->timestep << std::endl;
121	120	return (this->currentDate = this->getInitDate() + step * this->timestep);
122	121	}

XIOS/dev/branch_openmp/src/client.cpp

-                      r1205
+                      r1328
 #include "timer.hpp"
 #include "buffer_client.hpp"
+#include "log.hpp"
+using namespace ep_lib;
 namespace xios
+{
-    extern int test_omp_rank;
-    #pragma omp threadprivate(test_omp_rank)
     MPI_Comm CClient::intraComm ;
     MPI_Comm CClient::interComm ;
+    //std::list<MPI_Comm> CClient::contextInterComms;
     std::list<MPI_Comm> *CClient::contextInterComms_ptr = 0;
     int CClient::serverLeader ;
 …
     StdOFStream CClient::m_errorStream;
+    StdOFStream CClient::array_infoStream[16];
+    void CClient::initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)
+    void CClient::initialize(const string& codeId,MPI_Comm& localComm,MPI_Comm& returnComm)
+    {
       int initialized ;
 …
+      {
 // localComm doesn't given
         if (localComm == MPI_COMM_NULL)
+        {
           if (!is_MPI_Initialized)
+          {
+            //MPI_Init(NULL, NULL);
+            int return_level;
+            MPI_Init_thread(NULL, NULL, 3, &return_level);
+            assert(return_level == 3);
+            MPI_Init(NULL, NULL);
+          }
           CTimer::get("XIOS").resume() ;
 …
           int myColor ;
           int i,c ;
+          MPI_Comm_size(CXios::globalComm,&size);
+          MPI_Comm newComm ;
+          MPI_Comm_size(CXios::globalComm,&size) ;
           MPI_Comm_rank(CXios::globalComm,&rank);
           hashAll=new unsigned long[size] ;
 …
             MPI_Comm_size(intraComm,&intraCommSize) ;
             MPI_Comm_rank(intraComm,&intraCommRank) ;
+            #pragma omp critical(_output)
+            {
+              info(10)<<"intercommCreate::client "<<test_omp_rank<< " "<< &test_omp_rank <<" intraCommSize : "<<intraCommSize
+                 <<" intraCommRank :"<<intraCommRank<<"  serverLeader "<< serverLeader
+                 <<" globalComm : "<< &(CXios::globalComm) << endl ;
+            }
+            //test_sendrecv(CXios::globalComm);
+            info(50)<<"intercommCreate::client "<<rank<<" intraCommSize : "<<intraCommSize
+                 <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< serverLeader<<endl ;
             MPI_Intercomm_create(intraComm,0,CXios::globalComm,serverLeader,0,&interComm) ;
+          }
           else
 …
+      }
       // using OASIS
       else
+/*      else
+      {
         // localComm doesn't given
 …
         else MPI_Comm_dup(intraComm,&interComm) ;
+      }
+*/
       MPI_Comm_dup(intraComm,&returnComm) ;
+    }
 …
+    {
       CContext::setCurrent(id) ;
+      CContext* context = CContext::create(id);
+      int tmp_rank;
+      MPI_Comm_rank(contextComm,&tmp_rank) ;
+      CContext* context=CContext::create(id);
       StdString idServer(id);
       idServer += "_server";
 …
+      {
         int size,rank,globalRank ;
         //size_t message_size ;
         //int leaderRank ;
+        size_t message_size ;
+        int leaderRank ;
         MPI_Comm contextInterComm ;
 …
         CMessage msg ;
         msg<<idServer<<size<<globalRank ;
+//        msg<<id<<size<<globalRank ;
         int messageSize=msg.size() ;
 …
         MPI_Intercomm_create(contextComm,0,CXios::globalComm,serverLeader,10+globalRank,&contextInterComm) ;
+        #pragma omp critical(_output)
+        info(10)<<" RANK "<< tmp_rank<<" Register new Context : "<<id<<endl ;
+        info(10)<<"Register new Context : "<<id<<endl ;
         MPI_Comm inter ;
 …
         MPI_Barrier(inter) ;
         context->initClient(contextComm,contextInterComm) ;
+        //contextInterComms.push_back(contextInterComm);
         if(contextInterComms_ptr == NULL) contextInterComms_ptr = new std::list<MPI_Comm>;
         contextInterComms_ptr->push_back(contextInterComm);
         MPI_Comm_free(&inter);
+      }
 …
         // Finally, we should return current context to context client
         CContext::setCurrent(id);
+        //contextInterComms.push_back(contextInterComm);
         if(contextInterComms_ptr == NULL) contextInterComms_ptr = new std::list<MPI_Comm>;
         contextInterComms_ptr->push_back(contextInterComm);
+      }
+    }
 …
       MPI_Comm_rank(intraComm,&rank) ;
       if (!CXios::isServer)
+      {
 …
+      }
+      for (std::list<MPI_Comm>::iterator it = contextInterComms_ptr->begin(); it != contextInterComms_ptr->end(); ++it)
+      //for (std::list<MPI_Comm>::iterator it = contextInterComms.begin(); it != contextInterComms.end(); it++)
+      for (std::list<MPI_Comm>::iterator it = contextInterComms_ptr->begin(); it != contextInterComms_ptr->end(); it++)
         MPI_Comm_free(&(*it));
       MPI_Comm_free(&interComm);
       MPI_Comm_free(&intraComm);
 …
       if (!is_MPI_Initialized)
+      {
+        if (CXios::usingOasis) oasis_finalize();
+        else MPI_Finalize();
+        //if (CXios::usingOasis) oasis_finalize();
+        //else
+        MPI_Finalize() ;
+      }
+      #pragma omp critical (_output)
+      info(20) << "Client "<<rank<<" : Client side context is finalized "<< endl ;
+   /*#pragma omp critical (_output)
+   {
+      info(20) << "Client side context is finalized"<<endl ;
       report(0) <<" Performance report : Whole time from XIOS init and finalize: "<< CTimer::get("XIOS init/finalize").getCumulatedTime()<<" s"<<endl ;
       report(0) <<" Performance report : total time spent for XIOS : "<< CTimer::get("XIOS").getCumulatedTime()<<" s"<<endl ;
 …
       report(0)<< " Memory report : increasing it by a factor will increase performance, depending of the volume of data wrote in file at each time step of the file"<<endl ;
       report(100)<<CTimer::getAllCumulatedTime()<<endl ;
-   }*/
+   }
 …
       fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << getRank() << ext;
       fb->open(fileNameClient.str().c_str(), std::ios::out);
       if (!fb->is_open())
         ERROR("void CClient::openStream(const StdString& fileName, const StdString& ext, std::filebuf* fb)",
             << std::endl << "Can not open <" << fileNameClient << "> file to write the client log(s).");
+              << std::endl << "Can not open <" << fileNameClient << "> file to write the client log(s).");
+    }
 …
     void CClient::openInfoStream(const StdString& fileName)
+    {
+      //std::filebuf* fb = m_infoStream.rdbuf();
+      info_FB[omp_get_thread_num()] = array_infoStream[omp_get_thread_num()].rdbuf();
+      openStream(fileName, ".out", info_FB[omp_get_thread_num()]);
+      info.write2File(info_FB[omp_get_thread_num()]);
+      report.write2File(info_FB[omp_get_thread_num()]);
+      std::filebuf* fb = m_infoStream.rdbuf();
+      openStream(fileName, ".out", fb);
+      info.write2File(fb);
+      report.write2File(fb);
+    }

XIOS/dev/branch_openmp/src/client.hpp

-                      r1164
+                      r1328
+    {
       public:
         static void initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm);
+        static void initialize(const string& codeId, ep_lib::MPI_Comm& localComm, ep_lib::MPI_Comm& returnComm);
         static void finalize(void);
         static void registerContext(const string& id, ep_lib::MPI_Comm contextComm);
+        static MPI_Comm intraComm;
+        #pragma omp threadprivate(intraComm)
+        static MPI_Comm interComm;
+        #pragma omp threadprivate(interComm)
+        static ep_lib::MPI_Comm intraComm;
+        static ep_lib::MPI_Comm interComm;
         //static std::list<MPI_Comm> contextInterComms;
+        static std::list<MPI_Comm> * contextInterComms_ptr;
+        #pragma omp threadprivate(contextInterComms_ptr)
+        static std::list<ep_lib::MPI_Comm> *contextInterComms_ptr;
         static int serverLeader;
-        #pragma omp threadprivate(serverLeader)
         static bool is_MPI_Initialized ;
-        #pragma omp threadprivate(is_MPI_Initialized)
         //! Get rank of the current process
 …
       protected:
         static int rank;
-        #pragma omp threadprivate(rank)
         static StdOFStream m_infoStream;
-        #pragma omp threadprivate(m_infoStream)
         static StdOFStream m_errorStream;
-        #pragma omp threadprivate(m_errorStream)
-        static StdOFStream array_infoStream[16];
         static void openStream(const StdString& fileName, const StdString& ext, std::filebuf* fb);

XIOS/dev/branch_openmp/src/client_client_dht_template.hpp

-                      r1134
+                      r1328
 #include "xios_spl.hpp"
 #include "array_new.hpp"
 #include "mpi_std.hpp"
+#include "mpi.hpp"
 #include "policy.hpp"
 #include <boost/unordered_map.hpp>
 …
                            const ep_lib::MPI_Comm& clientIntraComm,
                            std::vector<ep_lib::MPI_Request>& requestSendInfo);
+    void sendInfoToClients(int clientDestRank, unsigned char* info, int infoSize,
+                           const ep_lib::MPI_Comm& clientIntraComm,
+                           ep_lib::MPI_Request* requestSendInfo);
     void recvInfoFromClients(int clientSrcRank, unsigned char* info, int infoSize,
                             const ep_lib::MPI_Comm& clientIntraComm,
                             std::vector<ep_lib::MPI_Request>& requestRecvInfo);
+    void recvInfoFromClients(int clientSrcRank, unsigned char* info, int infoSize,
+                             const ep_lib::MPI_Comm& clientIntraComm,
+                             ep_lib::MPI_Request* requestRecvInfo);
     // Send global index to clients
 …
                             const ep_lib::MPI_Comm& clientIntraComm,
                             std::vector<ep_lib::MPI_Request>& requestSendIndexGlobal);
+    void sendIndexToClients(int clientDestRank, size_t* indices, size_t indiceSize,
+                            const ep_lib::MPI_Comm& clientIntraComm,
+                            ep_lib::MPI_Request* requestSendIndexGlobal);
     void recvIndexFromClients(int clientSrcRank, size_t* indices, size_t indiceSize,
                              const ep_lib::MPI_Comm& clientIntraComm,
                              std::vector<ep_lib::MPI_Request>& requestRecvIndex);
+    void recvIndexFromClients(int clientSrcRank, size_t* indices, size_t indiceSize,
+                              const ep_lib::MPI_Comm& clientIntraComm,
+                              ep_lib::MPI_Request* requestRecvIndex);
     void sendRecvOnReturn(const std::vector<int>& sendNbRank, std::vector<int>& sendNbElements,

XIOS/dev/branch_openmp/src/client_client_dht_template_impl.hpp

-                      r1209
+                      r1328
 #include "utils.hpp"
 #include "mpi_tag.hpp"
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#include "ep_lib.hpp"
-#endif
 namespace xios
 …
   : H(clientIntraComm), index2InfoMapping_(), indexToInfoMappingLevel_(), nbClient_(0)
+{
   MPI_Comm_size(clientIntraComm, &nbClient_);
+  ep_lib::MPI_Comm_size(clientIntraComm, &nbClient_);
   this->computeMPICommLevel();
   int nbLvl = this->getNbLevel();
 …
   : H(clientIntraComm), index2InfoMapping_(), indexToInfoMappingLevel_(), nbClient_(0)
+{
   MPI_Comm_size(clientIntraComm, &nbClient_);
+  ep_lib::MPI_Comm_size(clientIntraComm, &nbClient_);
   this->computeMPICommLevel();
   int nbLvl = this->getNbLevel();
 …
   : H(clientIntraComm), index2InfoMapping_(), indexToInfoMappingLevel_(), nbClient_(0)
+{
   MPI_Comm_size(clientIntraComm, &nbClient_);
+  ep_lib::MPI_Comm_size(clientIntraComm, &nbClient_);
   this->computeMPICommLevel();
   int nbLvl = this->getNbLevel();
 …
+{
   int clientRank;
+  MPI_Comm_rank(commLevel,&clientRank);
+  //ep_lib::MPI_Barrier(commLevel);
+  ep_lib::MPI_Comm_rank(commLevel,&clientRank);
   int groupRankBegin = this->getGroupBegin()[level];
   int nbClient = this->getNbInGroup()[level];
 …
     recvIndexBuff = new unsigned long[recvNbIndexCount];
+  int request_size = 0;
+  int currentIndex = 0;
+  int nbRecvClient = recvRankClient.size();
+  int position = 0;
+  for (int idx = 0; idx < nbRecvClient; ++idx)
+int request_size = 0;
+  for (int idx = 0; idx < recvRankClient.size(); ++idx)
+  {
     if (0 != recvNbIndexClientCount[idx])
+    {
+      request_size++;
+    }
+      request_size ++;
+  }
   request_size += client2ClientIndex.size();
   std::vector<ep_lib::MPI_Request> request(request_size);
   std::vector<int>::iterator itbRecvIndex = recvRankClient.begin(), itRecvIndex,
                              iteRecvIndex = recvRankClient.end(),
                            itbRecvNbIndex = recvNbIndexClientCount.begin(),
                            itRecvNbIndex;
+  int currentIndex = 0;
+  int nbRecvClient = recvRankClient.size();
+  int request_position = 0;
+  for (int idx = 0; idx < nbRecvClient; ++idx)
+  {
+    if (0 != recvNbIndexClientCount[idx])
+      recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, &request[request_position++]);
+      //recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
+    currentIndex += recvNbIndexClientCount[idx];
+  }
   boost::unordered_map<int, size_t* >::iterator itbIndex = client2ClientIndex.begin(), itIndex,
                                                 iteIndex = client2ClientIndex.end();
   for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex)
+  {
+    MPI_Isend(itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], MPI_UNSIGNED_LONG, itIndex->first, MPI_DHT_INDEX, commLevel, &request[position]);
+    position++;
+    sendIndexToClients(itIndex->first, (itIndex->second), sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, &request[request_position++]);
     //sendIndexToClients(itIndex->first, (itIndex->second), sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
+  }
+  for (int idx = 0; idx < nbRecvClient; ++idx)
+  {
+    if (0 != recvNbIndexClientCount[idx])
+    {
+      MPI_Irecv(recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], MPI_UNSIGNED_LONG,
+            recvRankClient[idx], MPI_DHT_INDEX, commLevel, &request[position]);
+      position++;
+      //recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
+    }
+    currentIndex += recvNbIndexClientCount[idx];
+  }
+  std::vector<ep_lib::MPI_Status> status(request_size);
+  MPI_Waitall(request.size(), &request[0], &status[0]);
+  std::vector<ep_lib::MPI_Status> status(request.size());
+  ep_lib::MPI_Waitall(request.size(), &request[0], &status[0]);
   CArray<size_t,1>* tmpGlobalIndex;
 …
     --level;
     computeIndexInfoMappingLevel(*tmpGlobalIndex, this->internalComm_, level);
+  }
   else // Now, we are in the last level where necessary mappings are.
 …
+  }
   request_size = 0;
+int requestOnReturn_size=0;
   for (int idx = 0; idx < recvRankOnReturn.size(); ++idx)
+  {
     if (0 != recvNbIndexOnReturn[idx])
+    {
       request_size += 2;
+      requestOnReturn_size += 2;
+    }
+  }
 …
     if (0 != sendNbIndexOnReturn[idx])
+    {
+      request_size += 2;
+    }
+  }
+  std::vector<ep_lib::MPI_Request> requestOnReturn(request_size);
+      requestOnReturn_size += 2;
+    }
+  }
+  int requestOnReturn_position=0;
+  std::vector<ep_lib::MPI_Request> requestOnReturn(requestOnReturn_size);
   currentIndex = 0;
-  position = 0;
   for (int idx = 0; idx < recvRankOnReturn.size(); ++idx)
+  {
 …
+    {
       //recvIndexFromClients(recvRankOnReturn[idx], recvIndexBuffOnReturn+currentIndex, recvNbIndexOnReturn[idx], commLevel, requestOnReturn);
-      MPI_Irecv(recvIndexBuffOnReturn+currentIndex, recvNbIndexOnReturn[idx], MPI_UNSIGNED_LONG,
-            recvRankOnReturn[idx], MPI_DHT_INDEX, commLevel, &requestOnReturn[position]);
-      position++;
       //recvInfoFromClients(recvRankOnReturn[idx],
       //                    recvInfoBuffOnReturn+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
       //                    recvNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(),
       //                    commLevel, requestOnReturn);
+      MPI_Irecv(recvInfoBuffOnReturn+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
+                recvNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), MPI_CHAR,
+                recvRankOnReturn[idx], MPI_DHT_INFO, commLevel, &requestOnReturn[position]);
+      position++;
+      recvIndexFromClients(recvRankOnReturn[idx], recvIndexBuffOnReturn+currentIndex, recvNbIndexOnReturn[idx], commLevel, &requestOnReturn[requestOnReturn_position++]);
+      recvInfoFromClients(recvRankOnReturn[idx],
+                          recvInfoBuffOnReturn+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
+                          recvNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(),
+                          commLevel, &requestOnReturn[requestOnReturn_position++]);
+    }
     currentIndex += recvNbIndexOnReturn[idx];
 …
       //sendIndexToClients(rank, client2ClientIndexOnReturn[rank],
       //                   sendNbIndexOnReturn[idx], commLevel, requestOnReturn);
-      MPI_Isend(client2ClientIndexOnReturn[rank], sendNbIndexOnReturn[idx], MPI_UNSIGNED_LONG,
-            rank, MPI_DHT_INDEX, commLevel, &requestOnReturn[position]);
-      position++;
       //sendInfoToClients(rank, client2ClientInfoOnReturn[rank],
       //                  sendNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), commLevel, requestOnReturn);
+      MPI_Isend(client2ClientInfoOnReturn[rank], sendNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), MPI_CHAR,
+            rank, MPI_DHT_INFO, commLevel, &requestOnReturn[position]);
+      position++;
+      sendIndexToClients(rank, client2ClientIndexOnReturn[rank],
+                         sendNbIndexOnReturn[idx], commLevel, &requestOnReturn[requestOnReturn_position++]);
+      sendInfoToClients(rank, client2ClientInfoOnReturn[rank],
+                        sendNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), commLevel, &requestOnReturn[requestOnReturn_position++]);
+    }
     currentIndex += recvNbIndexClientCount[idx];
 …
   std::vector<ep_lib::MPI_Status> statusOnReturn(requestOnReturn.size());
   MPI_Waitall(requestOnReturn.size(), &requestOnReturn[0], &statusOnReturn[0]);
+  ep_lib::MPI_Waitall(requestOnReturn.size(), &requestOnReturn[0], &statusOnReturn[0]);
   Index2VectorInfoTypeMap indexToInfoMapping;
 …
+{
   int clientRank;
   MPI_Comm_rank(commLevel,&clientRank);
+  ep_lib::MPI_Comm_rank(commLevel,&clientRank);
   computeSendRecvRank(level, clientRank);
-  //ep_lib::MPI_Barrier(commLevel);
   int groupRankBegin = this->getGroupBegin()[level];
 …
+    {
       client2ClientIndex[indexClient + groupRankBegin][sendNbIndexBuff[indexClient]] = it->first;;
+  //          ProcessDHTElement<InfoType>::packElement(it->second, client2ClientInfo[indexClient + groupRankBegin], sendNbInfo[indexClient]);
       ProcessDHTElement<InfoType>::packElement(infoTmp[idx], client2ClientInfo[indexClient + groupRankBegin], sendNbInfo[indexClient]);
       ++sendNbIndexBuff[indexClient];
 …
   int recvNbIndexCount = 0;
   for (int idx = 0; idx < recvNbIndexClientCount.size(); ++idx)
+  {
     recvNbIndexCount += recvNbIndexClientCount[idx];
+  }
   unsigned long* recvIndexBuff;
 …
   // it will send a message to the correct clients.
   // Contents of the message are index and its corresponding informatioin
+  int request_size = 0;
+  int request_size = 0;
+   for (int idx = 0; idx < recvRankClient.size(); ++idx)
+   {
+     if (0 != recvNbIndexClientCount[idx])
+     {
+       request_size += 2;
+     }
+   }
+   request_size += client2ClientIndex.size();
+   request_size += client2ClientInfo.size();
+   std::vector<ep_lib::MPI_Request> request(request_size);
   int currentIndex = 0;
   int nbRecvClient = recvRankClient.size();
+  int current_pos = 0;
+  int request_position=0;
   for (int idx = 0; idx < nbRecvClient; ++idx)
+  {
     if (0 != recvNbIndexClientCount[idx])
+    {
+      request_size += 2;
+    }
+    //currentIndex += recvNbIndexClientCount[idx];
+  }
+  request_size += client2ClientIndex.size();
+  request_size += client2ClientInfo.size();
+  std::vector<ep_lib::MPI_Request> request(request_size);
+  //unsigned long* tmp_send_buf_long[client2ClientIndex.size()];
+  //unsigned char* tmp_send_buf_char[client2ClientInfo.size()];
+  int info_position = 0;
+  int index_position = 0;
+      //recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
+      //recvInfoFromClients(recvRankClient[idx],
+      //                    recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
+      //                    recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(),
+      //                    commLevel, request);
+        recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, &request[request_position++]);
+        recvInfoFromClients(recvRankClient[idx],
+                            recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
+                            recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(),
+                            commLevel, &request[request_position++]);
+    }
+    currentIndex += recvNbIndexClientCount[idx];
+  }
   boost::unordered_map<int, size_t* >::iterator itbIndex = client2ClientIndex.begin(), itIndex,
                                                 iteIndex = client2ClientIndex.end();
   for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex)
+  {
+    sendIndexToClients(itIndex->first, itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, &request[request_position++]);
     //sendIndexToClients(itIndex->first, itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
-    //tmp_send_buf_long[index_position] = new unsigned long[sendNbIndexBuff[itIndex->first-groupRankBegin]];
-    //for(int i=0; i<sendNbIndexBuff[itIndex->first-groupRankBegin]; i++)
-    //{
-    //  tmp_send_buf_long[index_position][i] = (static_cast<unsigned long * >(itIndex->second))[i];
-    //}
-    //MPI_Isend(tmp_send_buf_long[current_pos], sendNbIndexBuff[itIndex->first-groupRankBegin], MPI_UNSIGNED_LONG,
-    MPI_Isend(itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], MPI_UNSIGNED_LONG,
-              itIndex->first, MPI_DHT_INDEX, commLevel, &request[current_pos]);
-    current_pos++;
-    index_position++;
+  }
   boost::unordered_map<int, unsigned char*>::iterator itbInfo = client2ClientInfo.begin(), itInfo,
                                                       iteInfo = client2ClientInfo.end();
   for (itInfo = itbInfo; itInfo != iteInfo; ++itInfo)
+  {
+    sendInfoToClients(itInfo->first, itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], commLevel, &request[request_position++]);
     //sendInfoToClients(itInfo->first, itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], commLevel, request);
-    //tmp_send_buf_char[info_position] = new unsigned char[sendNbInfo[itInfo->first-groupRankBegin]];
-    //for(int i=0; i<sendNbInfo[itInfo->first-groupRankBegin]; i++)
-    //{
-    //  tmp_send_buf_char[info_position][i] = (static_cast<unsigned char * >(itInfo->second))[i];
-    //}
-    MPI_Isend(itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], MPI_CHAR,
-              itInfo->first, MPI_DHT_INFO, commLevel, &request[current_pos]);
-    current_pos++;
-    info_position++;
+  }
-  for (int idx = 0; idx < nbRecvClient; ++idx)
+  {
-    if (0 != recvNbIndexClientCount[idx])
+    {
-      //recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
-      MPI_Irecv(recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], MPI_UNSIGNED_LONG,
-                recvRankClient[idx], MPI_DHT_INDEX, commLevel, &request[current_pos]);
-      current_pos++;
-      MPI_Irecv(recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
-                recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(),
-                MPI_CHAR, recvRankClient[idx], MPI_DHT_INFO, commLevel, &request[current_pos]);
-      current_pos++;
-      // recvInfoFromClients(recvRankClient[idx],
-      //                     recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
-      //                     recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(),
-      //                     commLevel, request);
+    }
-    currentIndex += recvNbIndexClientCount[idx];
+  }
   std::vector<ep_lib::MPI_Status> status(request.size());
+  MPI_Waitall(request.size(), &request[0], &status[0]);
+  //for(int i=0; i<client2ClientInfo.size(); i++)
+  //  delete[] tmp_send_buf_char[i];
+  //for(int i=0; i<client2ClientIndex.size(); i++)
+  //  delete[] tmp_send_buf_long[i];
+  ep_lib::MPI_Waitall(request.size(), &request[0], &status[0]);
   Index2VectorInfoTypeMap indexToInfoMapping;
 …
   else
     index2InfoMapping_.swap(indexToInfoMapping);
+}
 …
                                                        std::vector<ep_lib::MPI_Request>& requestSendIndex)
+{
-  printf("should not call this function sendIndexToClients");
   ep_lib::MPI_Request request;
   requestSendIndex.push_back(request);
   MPI_Isend(indices, indiceSize, MPI_UNSIGNED_LONG,
+  ep_lib::MPI_Isend(indices, indiceSize, MPI_UNSIGNED_LONG,
             clientDestRank, MPI_DHT_INDEX, clientIntraComm, &(requestSendIndex.back()));
+}
+/*!
+  Send message containing index to clients
+  \param [in] clientDestRank rank of destination client
+  \param [in] indices index to send
+  \param [in] indiceSize size of index array to send
+  \param [in] clientIntraComm communication group of client
+  \param [in] requestSendIndex sending request
+*/
+template<typename T, typename H>
+void CClientClientDHTTemplate<T,H>::sendIndexToClients(int clientDestRank, size_t* indices, size_t indiceSize,
+                                                       const ep_lib::MPI_Comm& clientIntraComm,
+                                                       ep_lib::MPI_Request* requestSendIndex)
+{
+  ep_lib::MPI_Isend(indices, indiceSize, MPI_UNSIGNED_LONG,
+            clientDestRank, MPI_DHT_INDEX, clientIntraComm, requestSendIndex);
+}
 …
                                                          std::vector<ep_lib::MPI_Request>& requestRecvIndex)
+{
-  printf("should not call this function recvIndexFromClients");
   ep_lib::MPI_Request request;
   requestRecvIndex.push_back(request);
   MPI_Irecv(indices, indiceSize, MPI_UNSIGNED_LONG,
+  ep_lib::MPI_Irecv(indices, indiceSize, MPI_UNSIGNED_LONG,
             clientSrcRank, MPI_DHT_INDEX, clientIntraComm, &(requestRecvIndex.back()));
+}
+/*!
+  Receive message containing index to clients
+  \param [in] clientDestRank rank of destination client
+  \param [in] indices index to send
+  \param [in] clientIntraComm communication group of client
+  \param [in] requestRecvIndex receiving request
+*/
+template<typename T, typename H>
+void CClientClientDHTTemplate<T,H>::recvIndexFromClients(int clientSrcRank, size_t* indices, size_t indiceSize,
+                                                         const ep_lib::MPI_Comm& clientIntraComm,
+                                                         ep_lib::MPI_Request *requestRecvIndex)
+{
+  ep_lib::MPI_Irecv(indices, indiceSize, MPI_UNSIGNED_LONG,
+            clientSrcRank, MPI_DHT_INDEX, clientIntraComm, requestRecvIndex);
+}
 …
                                                       std::vector<ep_lib::MPI_Request>& requestSendInfo)
+{
-  printf("should not call this function sendInfoToClients");
   ep_lib::MPI_Request request;
   requestSendInfo.push_back(request);
+  MPI_Isend(info, infoSize, MPI_CHAR,
+  ep_lib::MPI_Isend(info, infoSize, MPI_CHAR,
             clientDestRank, MPI_DHT_INFO, clientIntraComm, &(requestSendInfo.back()));
+}
+/*!
+  Send message containing information to clients
+  \param [in] clientDestRank rank of destination client
+  \param [in] info info array to send
+  \param [in] infoSize info array size to send
+  \param [in] clientIntraComm communication group of client
+  \param [in] requestSendInfo sending request
+*/
+template<typename T, typename H>
+void CClientClientDHTTemplate<T,H>::sendInfoToClients(int clientDestRank, unsigned char* info, int infoSize,
+                                                      const ep_lib::MPI_Comm& clientIntraComm,
+                                                      ep_lib::MPI_Request *requestSendInfo)
+{
+  ep_lib::MPI_Isend(info, infoSize, MPI_CHAR,
+            clientDestRank, MPI_DHT_INFO, clientIntraComm, requestSendInfo);
+}
 …
                                                         std::vector<ep_lib::MPI_Request>& requestRecvInfo)
+{
-  printf("should not call this function recvInfoFromClients\n");
   ep_lib::MPI_Request request;
   requestRecvInfo.push_back(request);
   MPI_Irecv(info, infoSize, MPI_CHAR,
+  ep_lib::MPI_Irecv(info, infoSize, MPI_CHAR,
             clientSrcRank, MPI_DHT_INFO, clientIntraComm, &(requestRecvInfo.back()));
+}
+/*!
+  Receive message containing information from other clients
+  \param [in] clientDestRank rank of destination client
+  \param [in] info info array to receive
+  \param [in] infoSize info array size to receive
+  \param [in] clientIntraComm communication group of client
+  \param [in] requestRecvInfo list of receiving request
+*/
+template<typename T, typename H>
+void CClientClientDHTTemplate<T,H>::recvInfoFromClients(int clientSrcRank, unsigned char* info, int infoSize,
+                                                        const ep_lib::MPI_Comm& clientIntraComm,
+                                                        ep_lib::MPI_Request* requestRecvInfo)
+{
+  ep_lib::MPI_Irecv(info, infoSize, MPI_CHAR,
+            clientSrcRank, MPI_DHT_INFO, clientIntraComm, requestRecvInfo);
+}
 …
   int nRequest = 0;
+  for (int idx = 0; idx < recvNbRank.size(); ++idx)
+  {
+    ep_lib::MPI_Irecv(&recvNbElements[0]+idx, 1, MPI_INT,
+              recvNbRank[idx], MPI_DHT_INDEX_1, this->internalComm_, &request[nRequest]);
+    ++nRequest;
+  }
   for (int idx = 0; idx < sendNbRank.size(); ++idx)
+  {
     MPI_Isend(&sendNbElements[0]+idx, 1, MPI_INT,
+    ep_lib::MPI_Isend(&sendNbElements[0]+idx, 1, MPI_INT,
               sendNbRank[idx], MPI_DHT_INDEX_1, this->internalComm_, &request[nRequest]);
     ++nRequest;
+  }
+  for (int idx = 0; idx < recvNbRank.size(); ++idx)
+  {
+    MPI_Irecv(&recvNbElements[0]+idx, 1, MPI_INT,
+              recvNbRank[idx], MPI_DHT_INDEX_1, this->internalComm_, &request[nRequest]);
+    ++nRequest;
+  }
+  MPI_Waitall(sendNbRank.size()+recvNbRank.size(), &request[0], &requestStatus[0]);
+  ep_lib::MPI_Waitall(sendNbRank.size()+recvNbRank.size(), &request[0], &requestStatus[0]);
+}
 …
   std::vector<ep_lib::MPI_Request> request(sendBuffSize+recvBuffSize);
   std::vector<ep_lib::MPI_Status> requestStatus(sendBuffSize+recvBuffSize);
+  //ep_lib::MPI_Request request[sendBuffSize+recvBuffSize];
+  //ep_lib::MPI_Status requestStatus[sendBuffSize+recvBuffSize];
+  int my_rank;
+  MPI_Comm_rank(this->internalComm_, &my_rank);
   int nRequest = 0;
   for (int idx = 0; idx < recvBuffSize; ++idx)
+  {
     MPI_Irecv(&recvBuff[2*idx], 2, MPI_INT,
+    ep_lib::MPI_Irecv(&recvBuff[0]+2*idx, 2, MPI_INT,
               recvRank[idx], MPI_DHT_INDEX_0, this->internalComm_, &request[nRequest]);
     ++nRequest;
+  }
   for (int idx = 0; idx < sendBuffSize; ++idx)
 …
     sendBuff[idx*2+1] = sendNbElements[offSet];
+  }
   for (int idx = 0; idx < sendBuffSize; ++idx)
+  {
     MPI_Isend(&sendBuff[idx*2], 2, MPI_INT,
+    ep_lib::MPI_Isend(&sendBuff[idx*2], 2, MPI_INT,
               sendRank[idx], MPI_DHT_INDEX_0, this->internalComm_, &request[nRequest]);
     ++nRequest;
+  }
+  //MPI_Barrier(this->internalComm_);
+  MPI_Waitall(sendBuffSize+recvBuffSize, &request[0], &requestStatus[0]);
+  //MPI_Waitall(sendBuffSize+recvBuffSize, request, requestStatus);
+  ep_lib::MPI_Waitall(sendBuffSize+recvBuffSize, &request[0], &requestStatus[0]);
   int nbRecvRank = 0, nbRecvElements = 0;
   recvNbRank.clear();
 …
+    }
+  }
+}
+}
+}
+}

XIOS/dev/branch_openmp/src/client_server_mapping.cpp

-                      r1287
+                      r1328
  */
 #include "client_server_mapping.hpp"
+using namespace ep_lib;
 namespace xios {
 …
   MPI_Allgather(&nbConnectedServer,1,MPI_INT,recvCount,1,MPI_INT,clientIntraComm) ;
-  // for(int i=0; i<nbClient; i++)
-  //   printf("MPI_Allgather : recvCount[%d] = %d\n", i, recvCount[i]);
   displ[0]=0 ;
   for(int n=1;n<nbClient;n++) displ[n]=displ[n-1]+recvCount[n-1] ;
 …
   MPI_Allgatherv(sendBuff,nbConnectedServer,MPI_INT,recvBuff,recvCount,displ,MPI_INT,clientIntraComm) ;
-  // for(int i=0; i<recvSize; i++)
-  //   printf("MPI_Allgatherv : recvBuff[%d] = %d\n", i, recvBuff[i]);
   for(int n=0;n<recvSize;n++) clientRes[recvBuff[n]]++ ;

XIOS/dev/branch_openmp/src/client_server_mapping.hpp

-                      r1134
+                      r1328
 #include "mpi.hpp"
 #include <boost/unordered_map.hpp>
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 namespace xios {

XIOS/dev/branch_openmp/src/client_server_mapping_distributed.cpp

r907	r1328
15	15	#include "context.hpp"
16	16	#include "context_client.hpp"
	17	using namespace ep_lib;
17	18
18	19	namespace xios

XIOS/dev/branch_openmp/src/context_client.cpp

-                      r1205
+                      r1328
 #include "timer.hpp"
 #include "cxios.hpp"
+using namespace ep_lib;
 namespace xios
 …
     \cxtSer [in] cxtSer Pointer to context of server side. (It is only used on case of attached mode)
     */
     CContextClient::CContextClient(CContext* parent, ep_lib::MPI_Comm intraComm_, ep_lib::MPI_Comm interComm_, CContext* cxtSer)
+    CContextClient::CContextClient(CContext* parent, MPI_Comm intraComm_, MPI_Comm interComm_, CContext* cxtSer)
      : mapBufferSize_(), parentServer(cxtSer), maxBufferedEvents(4)
+    {
 …
        if (ratio < minBufferSizeEventSizeRatio) minBufferSizeEventSizeRatio = ratio;
+     }
+     #ifdef _usingMPI
+     MPI_Allreduce(MPI_IN_PLACE, &minBufferSizeEventSizeRatio, 1, MPI_DOUBLE, MPI_MIN, intraComm);
+     #elif _usingEP
+     //MPI_Allreduce(MPI_IN_PLACE, &minBufferSizeEventSizeRatio, 1, MPI_DOUBLE, MPI_MIN, intraComm);
      MPI_Allreduce(&minBufferSizeEventSizeRatio, &minBufferSizeEventSizeRatio, 1, MPI_DOUBLE, MPI_MIN, intraComm);
+     #endif
      if (minBufferSizeEventSizeRatio < 1.0)
+     {
 …
      for (itMap = itbMap; itMap != iteMap; ++itMap)
+     {
        //report(10) << " Memory report : Context <" << context->getId() << "> : client side : memory used for buffer of each connection to server" << endl
        //           << "  +) To server with rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
+       report(10) << " Memory report : Context <" << context->getId() << "> : client side : memory used for buffer of each connection to server" << endl
+                  << "  +) To server with rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
        totalBuf += itMap->second;
+     }
      //report(0) << " Memory report : Context <" << context->getId() << "> : client side : total memory used for buffer " << totalBuf << " bytes" << endl;
+     report(0) << " Memory report : Context <" << context->getId() << "> : client side : total memory used for buffer " << totalBuf << " bytes" << endl;
      releaseBuffers();

XIOS/dev/branch_openmp/src/context_server.cpp

-                      r1179
+                      r1328
 #include "file.hpp"
 #include "grid.hpp"
 #include "mpi_std.hpp"
+#include "mpi.hpp"
 #include "tracer.hpp"
 #include "timer.hpp"
 …
 #include <boost/functional/hash.hpp>
+using namespace ep_lib;
 namespace xios
+{
   CContextServer::CContextServer(CContext* parent, ep_lib::MPI_Comm intraComm_, ep_lib::MPI_Comm interComm_)
+  CContextServer::CContextServer(CContext* parent,MPI_Comm intraComm_,MPI_Comm interComm_)
+  {
     context=parent;
 …
     int count;
     char * addr;
     ep_lib::MPI_Status status;
+    MPI_Status status;
     map<int,CServerBuffer*>::iterator it;
+    for(rank=0;rank<commSize;rank++)
+    {
+    bool okLoop;
+    traceOff();
+    MPI_Iprobe(-2, 20,interComm,&flag,&status);
+    traceOn();
+    if (flag==true)
+    {
+      #ifdef _usingMPI
+      rank=status.MPI_SOURCE ;
+      #elif _usingEP
+      rank=status.ep_src ;
+      #endif
+      okLoop = true;
       if (pendingRequest.find(rank)==pendingRequest.end())
+      {
+        traceOff();
+        ep_lib::MPI_Iprobe(rank,20,interComm,&flag,&status);
+        traceOn();
+        if (flag)
+        okLoop = !listenPendingRequest(status) ;
+      if (okLoop)
+      {
+        for(rank=0;rank<commSize;rank++)
+        {
+          it=buffers.find(rank);
+          if (it==buffers.end()) // Receive the buffer size and allocate the buffer
+          if (pendingRequest.find(rank)==pendingRequest.end())
+          {
+            StdSize buffSize = 0;
+            ep_lib::MPI_Recv(&buffSize, 1, MPI_LONG, rank, 20, interComm, &status);
+            mapBufferSize_.insert(std::make_pair(rank, buffSize));
+            it=(buffers.insert(pair<int,CServerBuffer*>(rank,new CServerBuffer(buffSize)))).first;
+          }
+          else
+          {
+            ep_lib::MPI_Get_count(&status,MPI_CHAR,&count);
+            if (it->second->isBufferFree(count))
+            {
+              addr=(char*)it->second->getBuffer(count);
+              ep_lib::MPI_Irecv(addr,count,MPI_CHAR,rank,20,interComm,&pendingRequest[rank]);
+              bufferRequest[rank]=addr;
+            }
+            traceOff();
+            MPI_Iprobe(rank, 20,interComm,&flag,&status);
+            traceOn();
+            if (flag==true) listenPendingRequest(status) ;
+          }
+        }
 …
+  }
+  bool CContextServer::listenPendingRequest(MPI_Status& status)
+  {
+    int count;
+    char * addr;
+    map<int,CServerBuffer*>::iterator it;
+    #ifdef _usingMPI
+    int rank=status.MPI_SOURCE ;
+    #elif _usingEP
+    int rank=status.ep_src;
+    #endif
+    it=buffers.find(rank);
+    if (it==buffers.end()) // Receive the buffer size and allocate the buffer
+    {
+       StdSize buffSize = 0;
+       MPI_Recv(&buffSize, 1, MPI_LONG, rank, 20, interComm, &status);
+       mapBufferSize_.insert(std::make_pair(rank, buffSize));
+       it=(buffers.insert(pair<int,CServerBuffer*>(rank,new CServerBuffer(buffSize)))).first;
+       return true;
+    }
+    else
+    {
+      MPI_Get_count(&status,MPI_CHAR,&count);
+      if (it->second->isBufferFree(count))
+      {
+         addr=(char*)it->second->getBuffer(count);
+         MPI_Irecv(addr,count,MPI_CHAR,rank,20,interComm,&pendingRequest[rank]);
+         bufferRequest[rank]=addr;
+         return true;
+      }
+      else
+        return false;
+    }
+  }
   void CContextServer::checkPendingRequest(void)
+  {
     map<int,ep_lib::MPI_Request>::iterator it;
+    map<int,MPI_Request>::iterator it;
     list<int> recvRequest;
     list<int>::iterator itRecv;
 …
     int flag;
     int count;
     ep_lib::MPI_Status status;
     for(it=pendingRequest.begin();it!=pendingRequest.end();++it)
+    MPI_Status status;
+    for(it=pendingRequest.begin();it!=pendingRequest.end();it++)
+    {
       rank=it->first;
       traceOff();
       ep_lib::MPI_Test(& it->second, &flag, &status);
+      MPI_Test(& it->second, &flag, &status);
       traceOn();
       if (flag==true)
+      {
         recvRequest.push_back(rank);
         ep_lib::MPI_Get_count(&status,MPI_CHAR,&count);
+        MPI_Get_count(&status,MPI_CHAR,&count);
         processRequest(rank,bufferRequest[rank],count);
+      }
 …
+    {
       finished=true;
-      #pragma omp critical (_output)
       info(20)<<"Server Side context <"<<context->getId()<<"> finalized"<<endl;
       std::map<int, StdSize>::const_iterator itbMap = mapBufferSize_.begin(),
 …
       for (itMap = itbMap; itMap != iteMap; ++itMap)
+      {
         //report(10)<< " Memory report : Context <"<<context->getId()<<"> : server side : memory used for buffer of each connection to client" << endl
         //          << "  +) With client of rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
+        report(10)<< " Memory report : Context <"<<context->getId()<<"> : server side : memory used for buffer of each connection to client" << endl
+                  << "  +) With client of rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
         totalBuf += itMap->second;
+      }
       context->finalize();
       //report(0)<< " Memory report : Context <"<<context->getId()<<"> : server side : total memory used for buffer "<<totalBuf<<" bytes"<<endl;
+      report(0)<< " Memory report : Context <"<<context->getId()<<"> : server side : total memory used for buffer "<<totalBuf<<" bytes"<<endl;
+    }
     else if (event.classId==CContext::GetType()) CContext::dispatchEvent(event);

XIOS/dev/branch_openmp/src/context_server.hpp

r1134	r1328
17	17	bool eventLoop(bool enableEventsProcessing = true);
18	18	void listen(void) ;
	19	bool listenPendingRequest(ep_lib::MPI_Status& status);
19	20	void checkPendingRequest(void) ;
20	21	void processRequest(int rank, char* buff,int count) ;

XIOS/dev/branch_openmp/src/cxios.cpp

-                      r1205
+                      r1328
 #include "memtrack.hpp"
 #include "registry.hpp"
+using namespace ep_lib;
 namespace xios
+{
+  extern int test_omp_rank;
+  #pragma omp threadprivate(test_omp_rank)
+  const string CXios::rootFile="./iodef.xml" ;
+  const string CXios::xiosCodeId="xios.x" ;
+  const string CXios::clientFile="./xios_client";
+  const string CXios::serverFile="./xios_server";
+  string CXios::rootFile="./iodef.xml" ;
+  string CXios::xiosCodeId="xios.x" ;
+  string CXios::clientFile="./xios_client";
+  string CXios::serverFile="./xios_server";
   bool CXios::isClient ;
   bool CXios::isServer ;
   MPI_Comm CXios::globalComm ;
   bool CXios::usingOasis ;
   bool CXios::usingServer = false;
   double CXios::bufferSizeFactor = 1.0;
   const double CXios::defaultBufferSizeFactor = 1.0;
   StdSize CXios::minBufferSize = 1024 * sizeof(double);
   bool CXios::printLogs2Files;
   bool CXios::isOptPerformance = true;
 …
+  {
     set_new_handler(noMemory);
+    #pragma omp critical
+    {
+      parseFile(rootFile);
+    }
+    #pragma omp barrier
+    parseFile(rootFile);
     parseXiosConfig();
+  }
 …
       ERROR("CXios::parseXiosConfig()", "recv_field_timeout cannot be negative.");
+    //globalComm=MPI_COMM_WORLD ;
     int num_ep;
     if(isClient)
 …
       num_ep = omp_get_num_threads();
+    }
     if(isServer)
+    {
       num_ep = omp_get_num_threads();
+    }
     MPI_Info info;
     #pragma omp master
 …
       passage = ep_comm;
+    }
     #pragma omp barrier
     CXios::globalComm = passage[omp_get_thread_num()];
-    int tmp_rank;
-    MPI_Comm_rank(CXios::globalComm, &tmp_rank);
-    test_omp_rank = tmp_rank;
+  }
 …
     CClient::initialize(codeId,localComm,returnComm) ;
     if (CClient::getRank()==0) globalRegistry = new CRegistry(returnComm) ;
 …
     if (printLogs2Files)
+    {
-      #pragma omp critical
       CClient::openInfoStream(clientFile);
       CClient::openErrorStream(clientFile);
 …
      if (CClient::getRank()==0)
+     {
-       #pragma omp critical (_output)
        info(80)<<"Write data base Registry"<<endl<<globalRegistry->toString()<<endl ;
        globalRegistry->toFile("xios_registry.bin") ;
 …
   void CXios::initServer()
+  {
-    int initialized;
-    MPI_Initialized(&initialized);
-    if (initialized) CServer::is_MPI_Initialized=true ;
-    else CServer::is_MPI_Initialized=false ;
-    if(!CServer::is_MPI_Initialized)
+    {
-      MPI_Init(NULL, NULL);
+    }
     set_new_handler(noMemory);
     std::set<StdString> parseList;
 …
     initServer();
     // Initialize all aspects MPI
     CServer::initialize();

XIOS/dev/branch_openmp/src/cxios.hpp

-                      r1134
+                      r1328
 #include "mpi.hpp"
 #include "registry.hpp"
-#include "log.hpp"
 namespace xios
 …
+  {
     public:
       static void initialize(void) ;
       static void initClientSide(const string & codeId, ep_lib::MPI_Comm& localComm, ep_lib::MPI_Comm& returnComm) ;
       static void initServerSide(void) ;
       static void clientFinalize(void) ;
       static void parseFile(const string& filename) ;
+     static void initialize(void) ;
+     static void initClientSide(const string & codeId, ep_lib::MPI_Comm& localComm, ep_lib::MPI_Comm& returnComm) ;
+     static void initServerSide(void) ;
+     static void clientFinalize(void) ;
+     static void parseFile(const string& filename) ;
       template <typename T>
       static T getin(const string& id,const T& defaultValue) ;
+     template <typename T>
+     static T getin(const string& id,const T& defaultValue) ;
       template <typename T>
       static T getin(const string& id) ;
+     template <typename T>
+     static T getin(const string& id) ;
     public:
       static const string rootFile; //!< Configuration filename
       static const string xiosCodeId ; //!< Identity for XIOS
       static const string clientFile; //!< Filename template for client
       static const string serverFile; //!< Filename template for server
+     static string rootFile ; //!< Configuration filename
+     static string xiosCodeId ; //!< Identity for XIOS
+     static string clientFile; //!< Filename template for client
+     static string serverFile; //!< Filename template for server
+      static bool isClient ; //!< Check if xios is client
+      static bool isServer ; //!< Check if xios is server
+      #pragma omp threadprivate(isClient, isServer)
+     static bool isClient ; //!< Check if xios is client
+     static bool isServer ; //!< Check if xios is server
+      static MPI_Comm globalComm ; //!< Global communicator
+      #pragma omp threadprivate(globalComm)
+     static ep_lib::MPI_Comm globalComm ; //!< Global communicator
+      static bool printLogs2Files; //!< Printing out logs into files
+      static bool usingOasis ; //!< Using Oasis
+      static bool usingServer ; //!< Using server (server mode)
+      static double bufferSizeFactor; //!< Factor used to tune the buffer size
+      static const double defaultBufferSizeFactor; //!< Default factor value
+      static StdSize minBufferSize; //!< Minimum buffer size
+      static bool isOptPerformance; //!< Check if buffer size is for performance (as large as possible)
+      #pragma omp threadprivate(printLogs2Files, usingOasis, usingServer, bufferSizeFactor, minBufferSize, isOptPerformance)
+      static CRegistry* globalRegistry ; //!< global registry which is wrote by the root process of the servers
+      static double recvFieldTimeout;
+      #pragma omp threadprivate(recvFieldTimeout)
+     static bool printLogs2Files; //!< Printing out logs into files
+     static bool usingOasis ; //!< Using Oasis
+     static bool usingServer ; //!< Using server (server mode)
+     static double bufferSizeFactor; //!< Factor used to tune the buffer size
+     static const double defaultBufferSizeFactor; //!< Default factor value
+     static StdSize minBufferSize; //!< Minimum buffer size
+     static bool isOptPerformance; //!< Check if buffer size is for performance (as large as possible)
+     static CRegistry* globalRegistry ; //!< global registry which is wrote by the root process of the servers
+     static double recvFieldTimeout; //!< Time to wait for data before issuing an error when receiving a field
     public:
      //! Setting xios to use server mode

XIOS/dev/branch_openmp/src/data_output.cpp

r1205	r1328
4	4	#include "group_template.hpp"
5	5	#include "context.hpp"
6		//mpi.hpp
	6
7	7	namespace xios
8	8	{

XIOS/dev/branch_openmp/src/data_output.hpp

r1287	r1328
59	59	virtual void writeTimeDimension_(void) = 0;
60	60	virtual void writeTimeAxis_ (CField* field,
61		const ~~boost::~~shared_ptr<CCalendar> cal) = 0;
	61	const shared_ptr<CCalendar> cal) = 0;
62	62
63	63	/// PropriÃ©tÃ©s protÃ©gÃ©es ///

XIOS/dev/branch_openmp/src/dht_auto_indexing.cpp

-                      r1134
+                      r1328
  */
 #include "dht_auto_indexing.hpp"
+using namespace ep_lib;
 namespace xios
 …
   CDHTAutoIndexing::CDHTAutoIndexing(const CArray<size_t,1>& hashValue,
                                      const ep_lib::MPI_Comm& clientIntraComm)
+                                     const MPI_Comm& clientIntraComm)
     : CClientClientDHTTemplate<size_t>(clientIntraComm)
+  {
 …
   */
   CDHTAutoIndexing::CDHTAutoIndexing(Index2VectorInfoTypeMap& hashInitMap,
                                      const ep_lib::MPI_Comm& clientIntraComm)
+                                     const MPI_Comm& clientIntraComm)
     : CClientClientDHTTemplate<size_t>(clientIntraComm)
+  {

XIOS/dev/branch_openmp/src/dht_auto_indexing.hpp

-                      r1134
+                      r1328
 #include "client_client_dht_template.hpp"
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 namespace xios

XIOS/dev/branch_openmp/src/event_scheduler.cpp

-                      r1134
+                      r1328
 #include "xios_spl.hpp"
 #include "mpi.hpp"
+using namespace ep_lib;
 namespace xios
 …
     while(received)
+    {
+      #ifdef _usingEP
+      MPI_Iprobe(-1,1,communicator,&received, &status) ;
+      #else
+      MPI_Iprobe(MPI_ANY_SOURCE,1,communicator,&received, &status) ;
+      #endif
+      MPI_Iprobe(-2,1,communicator,&received, &status) ;
       if (received)
+      {
         recvRequest=new SPendingRequest ;
+        #ifdef _usingEP
+        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, -1, 1, communicator, &(recvRequest->request)) ;
+        #else
+        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 1, communicator, &(recvRequest->request)) ;
+        #endif
+        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, -2, 1, communicator, &(recvRequest->request)) ;
         pendingRecvParentRequest.push(recvRequest) ;
+      }
 …
     while(received)
+    {
+      #ifdef _usingEP
+      MPI_Iprobe(-1,0,communicator,&received, &status) ;
+      #else
+      MPI_Iprobe(MPI_ANY_SOURCE,0,communicator,&received, &status) ;
+      #endif
+      MPI_Iprobe(-2,0,communicator,&received, &status) ;
       if (received)
+      {
         recvRequest=new SPendingRequest ;
+        #ifdef _usingEP
+        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, -1, 0, communicator, &recvRequest->request) ;
+        #else
+        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, communicator, &recvRequest->request) ;
+        #endif
+        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, -2, 0, communicator, &recvRequest->request) ;
         pendingRecvChildRequest.push_back(recvRequest) ;
+      }

XIOS/dev/branch_openmp/src/event_scheduler.hpp

-                      r1134
+                      r1328
 #include "xios_spl.hpp"
 #include "mpi.hpp"
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 namespace xios
 …
         *  @param[in] comm : MPI communicator du duplicate for internal use
         */
        CEventScheduler(const MPI_Comm& comm) ;
+       CEventScheduler(const ep_lib::MPI_Comm& comm) ;
 …
+       {
          size_t buffer[3] ;      /*!< communication buffer : timeLine, hashId, level */
          MPI_Request request ;   /*!< pending MPI request */
+         ep_lib::MPI_Request request ;   /*!< pending MPI request */
        } ;
        MPI_Comm communicator ;  /*!< Internal MPI communicator */
+       ep_lib::MPI_Comm communicator ;  /*!< Internal MPI communicator */
        int mpiRank ;            /*!< Rank in the communicator */
        int mpiSize ;            /*!< Size of the communicator */

XIOS/dev/branch_openmp/src/filter/file_writer_filter.cpp

r1205	r1328
17	17	void CFileWriterFilter::onInputReady(std::vector<CDataPacketPtr> data)
18	18	{
19		~~CDataPacketPtr packet = data[0];~~
20
21	19	const bool detectMissingValue = (!field->detect_missing_value.isEmpty()
22	20	&& !field->default_value.isEmpty()
23	21	&& field->detect_missing_value == true);
	22
	23	CArray<double, 1> dataArray = (detectMissingValue) ? data[0]->data.copy() : data[0]->data;
	24
24	25	if (detectMissingValue)
25	26	{
26	27	const double missingValue = field->default_value;
27		const size_t nbData = ~~packet->data~~.numElements();
	28	const size_t nbData = dataArray.numElements();
28	29	for (size_t idx = 0; idx < nbData; ++idx)
29	30	{
30		if (NumTraits<double>::isnan(~~packet->data~~(idx)))
31		~~packet->data~~(idx) = missingValue;
	31	if (NumTraits<double>::isnan(dataArray(idx)))
	32	dataArray(idx) = missingValue;
32	33	}
33	34	}
34	35
35		field->sendUpdateData(~~packet->data~~);
	36	field->sendUpdateData(dataArray);
36	37	}
37	38

XIOS/dev/branch_openmp/src/filter/source_filter.cpp

r1205	r1328
99	99	void CSourceFilter::signalEndOfStream(CDate date)
100	100	{
	101	date = date + offset; // this is a temporary solution, it should be part of a proper temporal filter
	102
101	103	CDataPacketPtr packet(new CDataPacket);
102	104	packet->date = date;

XIOS/dev/branch_openmp/src/filter/spatial_transform_filter.cpp

-                      r1287
+                      r1328
+#include "mpi.hpp"
 #include "spatial_transform_filter.hpp"
 #include "grid_transformation.hpp"
 #include "context.hpp"
 #include "context_client.hpp"
+using namespace ep_lib;
 namespace xios
 …
+  }
+  //std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > CSpatialTransformFilterEngine::engines;
   std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > *CSpatialTransformFilterEngine::engines_ptr = 0;
 …
       ERROR("CSpatialTransformFilterEngine& CSpatialTransformFilterEngine::get(CGridTransformation* gridTransformation)",
             "Impossible to get the requested engine, the grid transformation is invalid.");
     if(engines_ptr == NULL) engines_ptr = new std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> >;
+    //std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> >::iterator it = engines.find(gridTransformation);
     std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> >::iterator it = engines_ptr->find(gridTransformation);
+    //if (it == engines.end())
     if (it == engines_ptr->end())
+    {
       boost::shared_ptr<CSpatialTransformFilterEngine> engine(new CSpatialTransformFilterEngine(gridTransformation));
+      //it = engines.insert(std::make_pair(gridTransformation, engine)).first;
       it = engines_ptr->insert(std::make_pair(gridTransformation, engine)).first;
+    }
 …
     double defaultValue = std::numeric_limits<double>::quiet_NaN();
     if (0 != dataDest.numElements()) ignoreMissingValue = NumTraits<double>::isnan(dataDest(0));
+    const std::list<CGridTransformation::SendingIndexGridSourceMap> *listLocalIndexSend_ptr = & (gridTransformation->getLocalIndexToSendFromGridSource());
     const std::list<CGridTransformation::SendingIndexGridSourceMap>& listLocalIndexSend = gridTransformation->getLocalIndexToSendFromGridSource();
     const std::list<CGridTransformation::RecvIndexGridDestinationMap>& listLocalIndexToReceive = gridTransformation->getLocalIndexToReceiveOnGridDest();
 …
     CArray<double,1> dataCurrentDest(dataSrc.copy());
     std::list<CGridTransformation::SendingIndexGridSourceMap>::const_iterator itListSend  = listLocalIndexSend_ptr->begin(),
                                                                               iteListSend = listLocalIndexSend_ptr->end();
+    std::list<CGridTransformation::SendingIndexGridSourceMap>::const_iterator itListSend  = listLocalIndexSend.begin(),
+                                                                              iteListSend = listLocalIndexSend.end();
     std::list<CGridTransformation::RecvIndexGridDestinationMap>::const_iterator itListRecv = listLocalIndexToReceive.begin();
     std::list<size_t>::const_iterator itNbListRecv = listNbLocalIndexToReceive.begin();
 …
           sendBuff[idxSendBuff] = new double[itSend->second.numElements()];
+      }
-      const CGridTransformation::RecvIndexGridDestinationMap& localIndexToReceive = *itListRecv;
-      CGridTransformation::RecvIndexGridDestinationMap::const_iterator itbRecv = localIndexToReceive.begin(), itRecv,
-                                                                       iteRecv = localIndexToReceive.end();
       idxSendBuff = 0;
       std::vector<ep_lib::MPI_Request> sendRecvRequest(localIndexToSend.size()+localIndexToReceive.size());
+      std::vector<MPI_Request> sendRecvRequest(localIndexToSend.size() + itListRecv->size());
       int position = 0;
       for (itSend = itbSend; itSend != iteSend; ++itSend, ++idxSendBuff)
 …
           sendBuff[idxSendBuff][idx] = dataCurrentSrc(localIndex_p(idx));
+        }
+        MPI_Isend(sendBuff[idxSendBuff], countSize, MPI_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest[position]);
+        position++;
+        MPI_Isend(sendBuff[idxSendBuff], countSize, MPI_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest[position++]);
+      }
       // Receiving data on destination fields
+      const CGridTransformation::RecvIndexGridDestinationMap& localIndexToReceive = *itListRecv;
+      CGridTransformation::RecvIndexGridDestinationMap::const_iterator itbRecv = localIndexToReceive.begin(), itRecv,
+                                                                       iteRecv = localIndexToReceive.end();
       int recvBuffSize = 0;
       for (itRecv = itbRecv; itRecv != iteRecv; ++itRecv) recvBuffSize += itRecv->second.size(); //(recvBuffSize < itRecv->second.size())
 …
         int srcRank = itRecv->first;
         int countSize = itRecv->second.size();
+        MPI_Irecv(recvBuff + currentBuff, countSize, MPI_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest[position]);
+        position++;
+        MPI_Irecv(recvBuff + currentBuff, countSize, MPI_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest[position++]);
         currentBuff += countSize;
+      }
       std::vector<ep_lib::MPI_Status> status(sendRecvRequest.size());
+      std::vector<MPI_Status> status(sendRecvRequest.size());
       MPI_Waitall(sendRecvRequest.size(), &sendRecvRequest[0], &status[0]);
 …
       std::vector<bool> localInitFlag(dataCurrentDest.numElements(), true);
       currentBuff = 0;
+      bool firstPass=true;
       for (itRecv = itbRecv; itRecv != iteRecv; ++itRecv)
+      {
 …
                          dataCurrentDest,
                          localInitFlag,
                          ignoreMissingValue);
+                         ignoreMissingValue,firstPass);
         currentBuff += countSize;
+        firstPass=false ;
+      }

XIOS/dev/branch_openmp/src/filter/spatial_transform_filter.hpp

-                      r1134
+                      r1328
 #include "filter.hpp"
 namespace xios
+{
 …
       //static std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > engines;
       static std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > *engines_ptr;
-      #pragma omp threadprivate(engines_ptr)
   }; // class CSpatialTransformFilterEngine
 } // namespace xios

XIOS/dev/branch_openmp/src/filter/store_filter.cpp

-                      r1205
+                      r1328
   void CStoreFilter::onInputReady(std::vector<CDataPacketPtr> data)
+  {
+    CDataPacketPtr packet = data[0];
+    CDataPacketPtr packet;
+    if (detectMissingValues)
+    {
+      const size_t nbData = data[0]->data.numElements();
+      packet = CDataPacketPtr(new CDataPacket);
+      packet->date = data[0]->date;
+      packet->timestamp = data[0]->timestamp;
+      packet->status = data[0]->status;
+      packet->data.resize(nbData);
+      packet->data = data[0]->data;
+      for (size_t idx = 0; idx < nbData; ++idx)
+      {
+        if (NumTraits<double>::isnan(packet->data(idx)))
+          packet->data(idx) = missingValue;
+      }
+    }
+    else
+    {
+      packet = data[0];
+    }
     packets.insert(std::make_pair(packet->timestamp, packet));
 …
     gc.registerObject(this, packet->timestamp);
-    if (detectMissingValues)
+    {
-      const size_t nbData = packet->data.numElements();
-      for (size_t idx = 0; idx < nbData; ++idx)
+      {
-        if (NumTraits<double>::isnan(packet->data(idx)))
-          packet->data(idx) = missingValue;
+      }
+    }
+  }

XIOS/dev/branch_openmp/src/filter/temporal_filter.cpp

-                      r1124
+                      r1328
     , isOnceOperation(functor->timeType() == func::CFunctor::once)
     , isInstantOperation(functor->timeType() == func::CFunctor::instant)
+    // If we can optimize the sampling when dealing with an instant functor we do it
+    , samplingFreq((isInstantOperation && samplingFreq == TimeStep && samplingOffset == NoneDu) ? opFreq : samplingFreq)
+    , samplingOffset((isInstantOperation && samplingFreq == TimeStep && samplingOffset == NoneDu) ? opFreq - initDate.getRelCalendar().getTimeStep() : samplingOffset)
+    , samplingFreq(samplingFreq)
+    , samplingOffset(samplingOffset)
     , opFreq(opFreq)
     , nextSamplingDate(initDate + this->samplingOffset + initDate.getRelCalendar().getTimeStep())

XIOS/dev/branch_openmp/src/group_factory.cpp

r1134	r1328
4	4	{
5	5	/// ////////////////////// DÃ©finitions ////////////////////// ///
6		StdString *CGroupFactory::CurrContext_ptr = new StdString;
	6	StdString CGroupFactory::CurrContext("");
7	7
8	8	void CGroupFactory::SetCurrentContextId(const StdString & context)
9		{
10		if(CGroupFactory::CurrContext_ptr == NULL ) CGroupFactory::CurrContext_ptr = new StdString;
11		CGroupFactory::CurrContext_ptr->assign(context);
	9	{
	10	CGroupFactory::CurrContext = context;
12	11	}
13	12
14	13	StdString & CGroupFactory::GetCurrentContextId(void)
15	14	{
16		return (*CGroupFactory::CurrContext_ptr);
	15	return (CGroupFactory::CurrContext);
17	16	}
18	17

XIOS/dev/branch_openmp/src/group_factory.hpp

-                      r1134
+                      r1328
          /// PropriÃ©tÃ©s statiques ///
+         static StdString *CurrContext_ptr;
+         #pragma omp threadprivate(CurrContext_ptr)
+         static StdString CurrContext;
    }; // class CGroupFactory

XIOS/dev/branch_openmp/src/group_factory_decl.cpp

-                      r1287
+                      r1328
+{
 # define  macro(U) \
   template void CGroupFactory::AddGroup<U>(boost::shared_ptr<U> pgroup,boost::shared_ptr<U> cgroup); \
   template void CGroupFactory::AddChild<U>(boost::shared_ptr<U> group, boost::shared_ptr<U::RelChild> child); \
   template boost::shared_ptr<U>  CGroupFactory::GetGroup<U>(boost::shared_ptr<U> group, const StdString & id); \
   template boost::shared_ptr<U::RelChild> CGroupFactory::GetChild<U>(boost::shared_ptr<U> group, const StdString & id); \
   template int CGroupFactory::GetGroupNum<U>(boost::shared_ptr<U> group); \
   template int CGroupFactory::GetGroupIdNum<U>(boost::shared_ptr<U> group); \
   template int CGroupFactory::GetChildNum<U>(boost::shared_ptr<U> group); \
+  template void CGroupFactory::AddGroup<U>(shared_ptr<U> pgroup,shared_ptr<U> cgroup); \
+  template void CGroupFactory::AddChild<U>(shared_ptr<U> group, shared_ptr<U::RelChild> child); \
+  template shared_ptr<U>  CGroupFactory::GetGroup<U>(shared_ptr<U> group, const StdString & id); \
+  template shared_ptr<U::RelChild> CGroupFactory::GetChild<U>(shared_ptr<U> group, const StdString & id); \
+  template int CGroupFactory::GetGroupNum<U>(shared_ptr<U> group); \
+  template int CGroupFactory::GetGroupIdNum<U>(shared_ptr<U> group); \
+  template int CGroupFactory::GetChildNum<U>(shared_ptr<U> group); \
   template int CGroupFactory::GetChildIdNum<U>(boost::shared_ptr<U> group); \
   template bool CGroupFactory::HasGroup<U>(boost::shared_ptr<U> group, const StdString & id); \
+  template bool CGroupFactory::HasGroup<U>(shared_ptr<U> group, const StdString & id); \
   template bool CGroupFactory::HasChild<U>(boost::shared_ptr<U> group, const StdString & id); \
   template boost::shared_ptr<U> CGroupFactory::CreateGroup<U>(boost::shared_ptr<U> group, const StdString & id ); \
   template boost::shared_ptr<U::RelChild>  CGroupFactory::CreateChild<U>(boost::shared_ptr<U> group, const StdString & id);
+  template shared_ptr<U> CGroupFactory::CreateGroup<U>(shared_ptr<U> group, const StdString & id ); \
+  template shared_ptr<U::RelChild>  CGroupFactory::CreateChild<U>(shared_ptr<U> group, const StdString & id);
   macro(CFieldGroup)

XIOS/dev/branch_openmp/src/indent.hpp

-                      r1134
+                      r1328
     public:
     static int defaultIncSize;
-    #pragma omp threadprivate(defaultIncSize)
     static int index ;
-    #pragma omp threadprivate(index)
     int incSize ;
     int offset ;

XIOS/dev/branch_openmp/src/indent_xml.cpp

r1134	r1328
15	15	{
16	16	static unsigned int LineNB = 1;
17		~~#pragma omp threadprivate(LineNB)~~
18
19	17	if (CIndent::WithLine) out << LineNB++ << ". ";
20	18	for(unsigned int i = 0; i < CIndent::Indent; out << CIndent::Increm , i++){}

XIOS/dev/branch_openmp/src/indent_xml.hpp

-                      r1134
+                      r1328
          /// PropriÃ©tÃ©s  statiques ///
          static unsigned int Indent;
-         #pragma omp threadprivate(Indent)
          static StdString    Increm;
-         #pragma omp threadprivate(Increm)
          static bool         WithLine;
-         #pragma omp threadprivate(WithLine)
    }; // class CIndent

XIOS/dev/branch_openmp/src/interface/c/icdata.cpp

-                      r1205
+                      r1328
 /* ************************************************************************** *
  *      Copyright Â© IPSL/LSCE, xios, Avril 2010 - Octobre 2011         *
+ *      Copyright  IPSL/LSCE, xios, Avril 2010 - Octobre 2011         *
  * ************************************************************************** */
 …
 #include <iostream>
+#include "mpi_std.hpp"
 #include "xios.hpp"
 //#include "oasis_cinterface.hpp"
 …
 #include "context.hpp"
 #include "context_client.hpp"
+#include "mpi_std.hpp"
 #include "timer.hpp"
-#include "array_new.hpp"
 extern "C"
+{
 // /////////////////////////////// DÃ©finitions ////////////////////////////// //
    // ----------------------- RedÃ©finition de types ----------------------------
+// /////////////////////////////// Dfinitions ////////////////////////////// //
+   // ----------------------- Redfinition de types ----------------------------
    typedef enum { NETCDF4 = 0 } XFileType;
 …
    typedef xios::CContext* XContextPtr;
    // -------------------- Traitement des donnÃ©es ------------------------------
+   // -------------------- Traitement des donnes ------------------------------
    // This function is not exported to the public Fortran interface,
 …
       int initialized;
       MPI_Initialized(&initialized);
+      #ifdef _usingEP
+      if (initialized) local_comm = ep_lib::EP_Comm_f2c(static_cast< int >(*f_local_comm));
+      else local_comm = MPI_COMM_NULL;
+      #else
+      #ifdef _usingMPI
       if (initialized) local_comm=MPI_Comm_f2c(*f_local_comm);
+      else local_comm = MPI_COMM_NULL;
+      else local_comm=MPI_COMM_NULL;
+      #elif _usingEP
+      ep_lib::fc_comm_map.clear();
+      if (initialized) local_comm=ep_lib::EP_Comm_f2c(static_cast<int>(*f_local_comm));
+      else local_comm=MPI_COMM_NULL;
       #endif
       CXios::initClientSide(str, local_comm, return_comm);
+      #ifdef _usingEP
+      *f_return_comm = ep_lib::EP_Comm_c2f(return_comm);
+      #else
+      *f_return_comm = MPI_Comm_c2f(return_comm);
+      #ifdef _usingMPI
+      *f_return_comm=MPI_Comm_c2f(return_comm);
+      #elif _usingEP
+      *f_return_comm=ep_lib::EP_Comm_c2f(return_comm);
       #endif
       CTimer::get("XIOS init").suspend();
       CTimer::get("XIOS").suspend();
 …
      CTimer::get("XIOS").resume();
      CTimer::get("XIOS init context").resume();
+     #ifdef _usingMPI
+     comm=MPI_Comm_f2c(*f_comm);
+     #elif _usingEP
      comm = ep_lib::EP_Comm_f2c(static_cast< int >(*f_comm));
+     CClient::registerContext(str,comm);
+     #endif
+     CClient::registerContext(str, comm);
      CTimer::get("XIOS init context").suspend();
      CTimer::get("XIOS").suspend();
 …
    // ---------------------- Ecriture des donnÃ©es ------------------------------
+   // ---------------------- Ecriture des donnes ------------------------------
    void cxios_write_data_k80(const char* fieldid, int fieldid_size, double* data_k8, int data_Xsize)
 …
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
+      {
+        context->checkBuffersAndListen();
+      }
+        context->checkBuffersAndListen();
       CArray<double, 3>data(data_k8, shape(data_Xsize, data_Ysize, data_Zsize), neverDeleteData);
 …
+    }
    // ---------------------- Lecture des donnÃ©es ------------------------------
+   // ---------------------- Lecture des donnes ------------------------------
    void cxios_read_data_k80(const char* fieldid, int fieldid_size, double* data_k8, int data_Xsize)

XIOS/dev/branch_openmp/src/interface/c/oasis_cinterface.cpp

-                      r1134
+                      r1328
 #include "oasis_cinterface.hpp"
 #include <string>
+//#include "mpi_std.hpp"
+using namespace ep_lib;
 namespace xios
 …
     fxios_oasis_get_localcomm(&f_comm) ;
-    #ifdef _usingEP
     comm=EP_Comm_f2c(f_comm.mpi_fint) ;
-    #else
-    comm=MPI_Comm_f2c(f_comm) ;
-    #endif
+  }
 …
     fxios_oasis_get_intracomm(&f_comm,server_id.data(),server_id.size()) ;
-    #ifdef _usingEP
     comm_client_server=EP_Comm_f2c(f_comm.mpi_fint) ;
-    #else
-    comm_client_server=MPI_Comm_f2c(f_comm) ;
-    #endif
+  }
 …
     fxios_oasis_get_intercomm(&f_comm,server_id.data(),server_id.size()) ;
-    #ifdef _usingEP
     comm_client_server=EP_Comm_f2c(f_comm.mpi_fint) ;
-    #else
-    comm_client_server=MPI_Comm_f2c(f_comm) ;
-    #endif
+  }
+}

XIOS/dev/branch_openmp/src/interface/c/oasis_cinterface.hpp

-                      r501
+                      r1328
   void fxios_oasis_enddef(void) ;
   void fxios_oasis_finalize(void) ;
   void fxios_oasis_get_localcomm(MPI_Fint* f_comm) ;
   void fxios_oasis_get_intracomm(MPI_Fint* f_comm_client_server,const char* client_id,int str_len) ;
   void fxios_oasis_get_intercomm(MPI_Fint* f_comm_client_server,const char* client_id,int str_len) ;
+  void fxios_oasis_get_localcomm(ep_lib::MPI_Fint* f_comm) ;
+  void fxios_oasis_get_intracomm(ep_lib::MPI_Fint* f_comm_client_server,const char* client_id,int str_len) ;
+  void fxios_oasis_get_intercomm(ep_lib::MPI_Fint* f_comm_client_server,const char* client_id,int str_len) ;
+}
 …
   void oasis_enddef(void) ;
   void oasis_finalize(void) ;
   void oasis_get_localcomm(MPI_Comm& comm) ;
   void oasis_get_intracomm(MPI_Comm& comm_client_server,const std::string& server_id) ;
   void oasis_get_intercomm(MPI_Comm& comm_client_server,const std::string& server_id) ;
+  void oasis_get_localcomm(ep_lib::MPI_Comm& comm) ;
+  void oasis_get_intracomm(ep_lib::MPI_Comm& comm_client_server,const std::string& server_id) ;
+  void oasis_get_intercomm(ep_lib::MPI_Comm& comm_client_server,const std::string& server_id) ;
+}
 #endif

XIOS/dev/branch_openmp/src/io/inetcdf4.cpp

-                      r1287
+                      r1328
 #include <boost/algorithm/string.hpp>
-// mpi_std.hpp
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 namespace xios
+{
   CINetCDF4::CINetCDF4(const StdString& filename, const MPI_Comm* comm /*= NULL*/, bool multifile /*= true*/, const StdString& timeCounterName /*= "time_counter"*/)
+  CINetCDF4::CINetCDF4(const StdString& filename, const ep_lib::MPI_Comm* comm /*= NULL*/, bool multifile /*= true*/, const StdString& timeCounterName /*= "time_counter"*/)
+  {
     // Don't use parallel mode if there is only one process
 …
+    {
       int commSize = 0;
       MPI_Comm_size(*comm, &commSize);
+      ep_lib::MPI_Comm_size(*comm, &commSize);
       if (commSize <= 1)
         comm = NULL;
+    }
     mpi = comm && !multifile;
     MPI_Info m_info = MPI_INFO_NULL.mpi_info;
+    ep_lib::MPI_Info info_null;
     // The file format will be detected automatically by NetCDF, it is safe to always set NC_MPIIO
     // even if Parallel NetCDF ends up being used.
     if (mpi)
+      CNetCdfInterface::openPar(filename, NC_NOWRITE | NC_MPIIO, *comm, m_info, this->ncidp);
+      //CNetCdfInterface::openPar(filename, NC_NOWRITE | NC_MPIIO, *comm, info_null, this->ncidp);
+      CNetCdfInterface::openPar(filename, NC_NOWRITE | NC_MPIIO, static_cast<MPI_Comm>(comm->mpi_comm), info_null.mpi_info, this->ncidp);
     else
       CNetCdfInterface::open(filename, NC_NOWRITE, this->ncidp);

XIOS/dev/branch_openmp/src/io/inetcdf4.hpp

-                      r1138
+                      r1328
 #endif  // UNLIMITED_DIM
 namespace xios
+{
 …
     public:
       /// Constructors ///
       CINetCDF4(const StdString& filename, const MPI_Comm* comm = NULL, bool multifile = true,
+      CINetCDF4(const StdString& filename, const ep_lib::MPI_Comm* comm = NULL, bool multifile = true,
                 const StdString& timeCounterName = "time_counter");
       CINetCDF4(const CINetCDF4& inetcdf4);       // Not implemented.

XIOS/dev/branch_openmp/src/io/inetcdf4_decl.cpp

r1138	r1328
1	1	#include "inetcdf4_impl.hpp"
2		~~// mpi_std.hpp~~
3	2
4	3	namespace xios

XIOS/dev/branch_openmp/src/io/inetcdf4_impl.hpp

r1138	r1328
4	4	#include "inetcdf4.hpp"
5	5	#include "netCdfInterface.hpp"
6		~~// mpi_std.hpp~~
7	6
8	7	namespace xios

XIOS/dev/branch_openmp/src/io/nc4_data_input.cpp

-                      r1176
+                      r1328
 #include "scalar.hpp"
-// mpi.hpp
 namespace xios
+{
   CNc4DataInput::CNc4DataInput(const StdString& filename, ::MPI_Comm comm_file, bool multifile, bool isCollective /*= true*/, const StdString& timeCounterName /*= "time_counter"*/)
+  CNc4DataInput::CNc4DataInput(const StdString& filename, ep_lib::MPI_Comm comm_file, bool multifile, bool isCollective /*= true*/, const StdString& timeCounterName /*= "time_counter"*/)
     : SuperClass()
     , SuperClassWriter(filename, &comm_file, multifile, timeCounterName)
 …
     CArray<double,1> fieldData(grid->getWrittenDataSize());
     if (!field->default_value.isEmpty()) fieldData = field->default_value;
+    #ifdef _usingEP
+    SuperClass::type = ONE_FILE;
+    printf("SuperClass::type = %d\n", SuperClass::type);
+    #endif
     switch (SuperClass::type)
+    {
 …
       std::vector<StdSize> nBeginBndsLatLon(3), nSizeBndsLatLon(3);
       nBeginBndsLatLon[0] = 0; nSizeBndsLatLon[0] = domain->nj_glo.getValue();
       nBeginBndsLatLon[1] = 0; nSizeBndsLatLon[1] = domain->nj_glo.getValue();
+      nBeginBndsLatLon[1] = 0; nSizeBndsLatLon[1] = domain->ni_glo.getValue();
       nBeginBndsLatLon[2] = 0; nSizeBndsLatLon[2] = nbVertex;

XIOS/dev/branch_openmp/src/io/nc4_data_input.hpp

-                      r1138
+                      r1328
 /// XIOS headers ///
+#include "mpi_std.hpp"
 #include "xios_spl.hpp"
 #include "data_input.hpp"
 #include "inetcdf4.hpp"
-// mpi_std.hpp
 namespace xios
 …
     /// Constructors ///
     CNc4DataInput(const StdString& filename, MPI_Comm comm_file, bool multifile, bool isCollective = true,
+    CNc4DataInput(const StdString& filename, ep_lib::MPI_Comm comm_file, bool multifile, bool isCollective = true,
                   const StdString& timeCounterName = "time_counter");
     CNc4DataInput(const CNc4DataInput& dataInput);       // Not implemented.
 …
   private:
     /// Private attributes ///
     MPI_Comm comm_file;
+    ep_lib::MPI_Comm comm_file;
     const StdString filename;
     bool isCollective;

XIOS/dev/branch_openmp/src/io/nc4_data_output.cpp

-                      r1205
+                      r1328
 #include "timer.hpp"
 #include "uuid.hpp"
-// mpi.hpp
 namespace xios
+{
 …
       CNc4DataOutput::CNc4DataOutput
          (CFile* file, const StdString & filename, bool exist, bool useClassicFormat, bool useCFConvention,
           ::MPI_Comm comm_file, bool multifile, bool isCollective, const StdString& timeCounterName)
+          ep_lib::MPI_Comm comm_file, bool multifile, bool isCollective, const StdString& timeCounterName)
             : SuperClass()
             , SuperClassWriter(filename, exist, useClassicFormat, useCFConvention, &comm_file, multifile, timeCounterName)
 …
       StdString domainName = domain->name;
       domain->assignMesh(domainName, domain->nvertex);
       domain->mesh->createMeshEpsilon(static_cast< ::MPI_Comm >(server->intraComm.mpi_comm), domain->lonvalue_srv, domain->latvalue_srv, domain->bounds_lon_srv, domain->bounds_lat_srv);
+      domain->mesh->createMeshEpsilon(server->intraComm, domain->lonvalue_srv, domain->latvalue_srv, domain->bounds_lon_srv, domain->bounds_lat_srv);
       StdString node_x = domainName + "_node_x";

XIOS/dev/branch_openmp/src/io/nc4_data_output.hpp

-                      r1138
+                      r1328
 /// XIOS headers ///
 #include "xios_spl.hpp"
+#include "mpi_std.hpp"
 #include "onetcdf4.hpp"
 #include "data_output.hpp"
-// mpi_std.hpp
 namespace xios
 …
             CNc4DataOutput
                (CFile* file, const StdString & filename, bool exist, bool useClassicFormat,
+                bool useCFConvention, MPI_Comm comm_file, bool multifile,
+                bool isCollective = true, const StdString& timeCounterName = "time_counter");
+                bool useCFConvention,
+                ep_lib::MPI_Comm comm_file, bool multifile, bool isCollective = true,
+                const StdString& timeCounterName = "time_counter");
             CNc4DataOutput(const CNc4DataOutput & dataoutput);       // Not implemented.
 …
             /// PropriÃ©tÃ©s privÃ©es ///
             MPI_Comm comm_file;
+            ep_lib::MPI_Comm comm_file;
             const StdString filename;
             std::map<Time, StdSize> timeToRecordCache;

XIOS/dev/branch_openmp/src/io/netCdfInterface.cpp

r1153	r1328
10	10	#include "netCdfInterface.hpp"
11	11	#include "netCdfException.hpp"
12		~~// mpi_std.hpp~~
13	12
14	13	namespace xios
…	…
50	49	int CNetCdfInterface::createPar(const StdString& fileName, int cMode, MPI_Comm comm, MPI_Info info, int& ncId)
51	50	{
52		int status = xios::nc_create_par(fileName.c_str(), cMode, comm, info, &ncId);
	51	int status = xios::nc_create_par(fileName.c_str(), cMode, comm, MPI_INFO_NULL.mpi_info, &ncId);
53	52	if (NC_NOERR != status)
54	53	{
…	…
75	74	int CNetCdfInterface::open(const StdString& fileName, int oMode, int& ncId)
76	75	{
77		int status = NC_NOERR;
78		#pragma omp critical (_netcdf)
79		status = nc_open(fileName.c_str(), oMode, &ncId);
80
	76	int status = nc_open(fileName.c_str(), oMode, &ncId);
81	77	if (NC_NOERR != status)
82	78	{
…	…
106	102	int CNetCdfInterface::openPar(const StdString& fileName, int oMode, MPI_Comm comm, MPI_Info info, int& ncId)
107	103	{
108		int status;
109		#pragma omp critical (_netcdf)
110		status = xios::nc_open_par(fileName.c_str(), oMode, comm, info, &ncId); // nc_open
	104	//int status = xios::nc_open_par(fileName.c_str(), oMode, comm, info, &ncId);
	105	int status = xios::nc_open_par(fileName.c_str(), oMode, comm, MPI_INFO_NULL.mpi_info, &ncId);
111	106	if (NC_NOERR != status)
112	107	{
…	…
131	126	int CNetCdfInterface::close(int ncId)
132	127	{
133		int status = NC_NOERR;
134		#pragma omp critical (_netcdf)
135		//#pragma omp master
136		{
137		status = nc_close(ncId);
	128	int status = nc_close(ncId);
138	129	if (NC_NOERR != status)
139	130	{
…	…
146	137	throw CNetCdfException(e);
147	138	}
148		}
	139
149	140	return status;
150	141	}
…	…
356	347	int CNetCdfInterface::inqDimLen(int ncid, int dimId, StdSize& dimLen)
357	348	{
358		int status;
359		#pragma omp critical (_netcdf)
360		status = nc_inq_dimlen(ncid, dimId, &dimLen);
	349	int status = nc_inq_dimlen(ncid, dimId, &dimLen);
361	350	if (NC_NOERR != status)
362	351	{

XIOS/dev/branch_openmp/src/io/netCdfInterface.hpp

-                      r1134
+                      r1328
 #define __NETCDF_INTERFACE_HPP_
+#include "mpi_std.hpp"
 #include "xios_spl.hpp"
 …
 #endif
-#include "mpi_std.hpp"
 #include "netcdf.hpp"

XIOS/dev/branch_openmp/src/io/netCdfInterface_decl.cpp

r1138	r1328
9	9
10	10	#include "netCdfInterface_impl.hpp"
11		~~// mpi_std.hpp~~
12	11
13	12	namespace xios

XIOS/dev/branch_openmp/src/io/netCdfInterface_impl.hpp

-                      r1146
+                      r1328
 #include "netCdfInterface.hpp"
 #include "netCdfException.hpp"
-// mpi_std.hpp
 namespace xios
 …
   int CNetCdfInterface::getVaraType(int ncid, int varId, const StdSize* start, const StdSize* count, T* data)
+  {
+    int status;
+    #pragma omp critical (_netcdf)
+    status = ncGetVaraType(ncid, varId, start, count, data);
+    int status = ncGetVaraType(ncid, varId, start, count, data);
     if (NC_NOERR != status)
+    {

XIOS/dev/branch_openmp/src/io/netCdf_cf_constant.hpp

r1138	r1328
4	4	#include "inetcdf4.hpp"
5	5	#include "netCdfInterface.hpp"
6
7		~~// mpi_std.hpp~~
8	6
9	7	namespace xios

XIOS/dev/branch_openmp/src/io/netcdf.hpp

r1138	r1328
18	18	extern "C"
19	19	{
20		#include <netcdf_par.h>
	20	# include <netcdf_par.h>
21	21	}
22	22	# endif

XIOS/dev/branch_openmp/src/io/onetcdf4.cpp

-                      r1287
+                      r1328
       /// ////////////////////// DÃ©finitions ////////////////////// ///
+      CONetCDF4::CONetCDF4(const StdString& filename, bool append, bool useClassicFormat, bool useCFConvention,
+                           const MPI_Comm* comm, bool multifile, const StdString& timeCounterName)
+      CONetCDF4::CONetCDF4(const StdString& filename, bool append, bool useClassicFormat,
+                                                        bool useCFConvention,
+                           const ep_lib::MPI_Comm* comm, bool multifile, const StdString& timeCounterName)
         : path()
         , wmpi(false)
 …
       void CONetCDF4::initialize(const StdString& filename, bool append, bool useClassicFormat, bool useCFConvention,
                                  const MPI_Comm* comm, bool multifile, const StdString& timeCounterName)
+                                 const ep_lib::MPI_Comm* comm, bool multifile, const StdString& timeCounterName)
+      {
          this->useClassicFormat = useClassicFormat;
 …
+         {
             int commSize = 0;
             MPI_Comm_size(*comm, &commSize);
+            ep_lib::MPI_Comm_size(*comm, &commSize);
             if (commSize <= 1)
                comm = NULL;
+         }
          wmpi = comm && !multifile;
+         ep_lib::MPI_Info info_null;
          if (wmpi)
 …
             CTimer::get("Files : create").resume();
             if (wmpi)
                CNetCdfInterface::createPar(filename, mode, *comm, MPI_INFO_NULL.mpi_info, this->ncidp);
+               CNetCdfInterface::createPar(filename, mode, static_cast<MPI_Comm>(comm->mpi_comm), info_null.mpi_info, this->ncidp);
             else
                CNetCdfInterface::create(filename, mode, this->ncidp);
 …
             CTimer::get("Files : open").resume();
             if (wmpi)
                CNetCdfInterface::openPar(filename, mode, *comm, MPI_INFO_NULL.mpi_info, this->ncidp);
+               CNetCdfInterface::openPar(filename, mode, static_cast<MPI_Comm>(comm->mpi_comm), info_null.mpi_info, this->ncidp);
             else
                CNetCdfInterface::open(filename, mode, this->ncidp);
 …
                                  const std::vector<StdSize>& scount, const int* data)
+      {
+         CNetCdfInterface::putVaraType(grpid, varid, &sstart[0], &scount[0], data);
+      }
+          CNetCdfInterface::putVaraType(grpid, varid, &sstart[0], &scount[0], data);
+      }
       //---------------------------------------------------------------
 …
                                  const std::vector<StdSize>& scount, const float* data)
+      {
          CNetCdfInterface::putVaraType(grpid, varid, &sstart[0], &scount[0], data);
+          CNetCdfInterface::putVaraType(grpid, varid, &sstart[0], &scount[0], data);
+      }

XIOS/dev/branch_openmp/src/io/onetcdf4.hpp

-                      r1205
+                      r1328
 /// XIOS headers ///
 #include "xios_spl.hpp"
+#include "mpi_std.hpp"
 #include "exception.hpp"
 #include "data_output.hpp"
 #include "array_new.hpp"
-#include "mpi_std.hpp"
 #include "netcdf.hpp"
 …
    #define UNLIMITED_DIM (size_t)(-1)
 #endif  //UNLIMITED_DIM
-// mpi_std.hpp
 namespace xios
 …
             CONetCDF4(const StdString& filename, bool append, bool useClassicFormat = false,
                           bool useCFConvention = true,
                       const MPI_Comm* comm = NULL, bool multifile = true,
+                      const ep_lib::MPI_Comm* comm = NULL, bool multifile = true,
                       const StdString& timeCounterName = "time_counter");
 …
             /// Initialisation ///
             void initialize(const StdString& filename, bool append, bool useClassicFormat, bool useCFConvention,
                             const MPI_Comm* comm, bool multifile, const StdString& timeCounterName);
+                            const ep_lib::MPI_Comm* comm, bool multifile, const StdString& timeCounterName);
             void close(void);
             void sync(void);

XIOS/dev/branch_openmp/src/io/onetcdf4_decl.cpp

r1138	r1328
1	1	#include "onetcdf4_impl.hpp"
2		~~// mpi_std.hpp~~
3	2
4	3	namespace xios

XIOS/dev/branch_openmp/src/io/onetcdf4_impl.hpp

r1205	r1328
5	5	#include "netCdfInterface.hpp"
6	6	#include "timer.hpp"
7		~~// mpi_std.hpp~~
8	7
9	8	namespace xios

XIOS/dev/branch_openmp/src/log.cpp

-                      r1164
+                      r1328
 #include "log.hpp"
-#include <string>
-#include <iostream>
-#include <string>
 namespace xios
+{
-  std::filebuf* info_FB[16];
   CLog info("info") ;
   CLog report("report") ;
   CLog error("error", cerr.rdbuf()) ;
-  CLog& CLog::operator()(int l)
+    {
-      if (l<=level)
+      {
-        omp_set_lock( &mutex );
-        //rdbuf(strBuf_);
-        rdbuf(strBuf_array[omp_get_thread_num()]);
-        *this<<"-> "<<name<<" : " ;
-        omp_unset_lock( &mutex );
+      }
-      else rdbuf(NULL) ;
-      return *this;
+    }
-  int test_omp_rank;
-  #pragma omp threadprivate(test_omp_rank)
+}

XIOS/dev/branch_openmp/src/log.hpp

-                      r1164
+                      r1328
 #include <iostream>
 #include <string>
-#include <stdio.h>
-#include <omp.h>
 namespace xios
 …
     public :
     CLog(const string& name_, std::streambuf* sBuff = cout.rdbuf())
+      : ostream(cout.rdbuf()), level(0), name(name_), strBuf_(sBuff)
+      : ostream(sBuff), level(0), name(name_), strBuf_(sBuff) {}
+    CLog& operator()(int l)
+    {
+      omp_init_lock( &mutex );
+      for(int i=0; i<16; i++)
+        strBuf_array[i] = sBuff;
+      if (l<=level)
+      {
+        rdbuf(strBuf_);
+        *this<<"-> "<<name<<" : " ;
+      }
+      else rdbuf(NULL) ;
+      return *this;
+    }
-    ~CLog()
+    {
-      omp_destroy_lock( &mutex );
+    }
-    CLog& operator()(int l);
     void setLevel(int l) {level=l; }
     int  getLevel() {return level ;}
+    int getLevel() {return level ;}
     bool isActive(void) { if (rdbuf()==NULL) return true ; else return false ;}
     bool isActive(int l) {if (l<=level) return true ; else return false ; }
 …
      * \param [in] pointer to new streambuf
     */
+    void changeStreamBuff(std::streambuf* sBuff)
+    {
+      strBuf_ = sBuff;
+      strBuf_array[omp_get_thread_num()] = sBuff;
+      rdbuf(sBuff);
+    }
+    void changeStreamBuff(std::streambuf* sBuff) { strBuf_ = sBuff; rdbuf(sBuff); }
     int level ;
     string name ;
     std::streambuf* strBuf_;
-    std::streambuf* strBuf_array[16];
-    omp_lock_t mutex;
   };
 …
   extern CLog report;
   extern CLog error;
-  extern std::filebuf* info_FB[16];
+}
 #endif

XIOS/dev/branch_openmp/src/memtrack.cpp

-                      r1205
+                      r1328
 #include <sstream>
 #include <string>
 #include <execinfo.h>
 …
         private:    // static member variables
             static BlockHeader *ourFirstNode;
-            #pragma omp threadprivate(ourFirstNode)
         private:    // member variables

XIOS/dev/branch_openmp/src/mpi.hpp

-                      r1134
+                      r1328
 /* skip C++ Binding for OpenMPI */
 #define OMPI_SKIP_MPICXX
 #ifdef _usingEP
   #include <omp.h>
   #include "../extern/src_ep_dev/ep_lib.hpp"
+  using namespace ep_lib;
+  #include "../extern/src_ep_dev/ep_declaration.hpp"
+  //using namespace ep_lib;
 #elif _usingMPI
   #include <mpi.h>

XIOS/dev/branch_openmp/src/mpi_std.hpp

-                      r1134
+                      r1328
 #define OMPI_SKIP_MPICXX
+#include <mpi.h>
 #include "../extern/src_ep_dev/ep_lib.hpp"
+#include <mpi.h>
+#include "../extern/src_ep_dev/ep_declaration.hpp"
 #endif

XIOS/dev/branch_openmp/src/node/axis.cpp

-                      r1205
+                      r1328
    { /* Ne rien faire de plus */ }
+   std::map<StdString, ETranformationType> *CAxis::transformationMapList_ptr = 0; //new std::map<StdString, ETranformationType>();
+   //bool CAxis::dummyTransformationMapList_ = CAxis::initializeTransformationMap(CAxis::transformationMapList_ptr);
+   //std::map<StdString, ETranformationType> CAxis::transformationMapList_ = std::map<StdString, ETranformationType>();
+   std::map<StdString, ETranformationType> *CAxis::transformationMapList_ptr = 0;
+   //bool CAxis::dummyTransformationMapList_ = CAxis::initializeTransformationMap(CAxis::transformationMapList_);
    bool CAxis::initializeTransformationMap(std::map<StdString, ETranformationType>& m)
+   {
 …
      m["extract_domain"] = TRANS_EXTRACT_DOMAIN_TO_AXIS;
+   }
    bool CAxis::initializeTransformationMap()
 …
      (*CAxis::transformationMapList_ptr)["extract_domain"]   = TRANS_EXTRACT_DOMAIN_TO_AXIS;
+   }
    ///---------------------------------------------------------------
 …
         globalAxisZoom[nZoomCount] = globalZoomIndex;
         ++nZoomCount;
+      }
+      }
+    }
 …
       CContextServer* server = CContext::getCurrent()->server;
       axis->numberWrittenIndexes_ = axis->indexesToWrite.size();
       ep_lib::MPI_Allreduce(&axis->numberWrittenIndexes_, &axis->totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
       ep_lib::MPI_Scan(&axis->numberWrittenIndexes_, &axis->offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      MPI_Allreduce(&axis->numberWrittenIndexes_, &axis->totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      MPI_Scan(&axis->numberWrittenIndexes_, &axis->offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
       axis->offsetWrittenIndexes_ -= axis->numberWrittenIndexes_;
+    }
 …
         msg << ni << begin << end;
         msg << global_zoom_begin.getValue() << global_zoom_n.getValue();
         msg << isCompressible_;
+        msg << isCompressible_;
         msg << zoomIndex;
         if (zoomIndex)
 …
         nodeElementName = node.getElementName();
+        //std::map<StdString, ETranformationType>::const_iterator ite = transformationMapList_.end(), it;
         if(transformationMapList_ptr == 0) initializeTransformationMap();
+        //transformationMapList_ptr = new std::map<StdString, ETranformationType>();
+        std::map<StdString, ETranformationType>::const_iterator ite = (*CAxis::transformationMapList_ptr).end(), it;
+        it = (*CAxis::transformationMapList_ptr).find(nodeElementName);
+        std::map<StdString, ETranformationType>::const_iterator ite = transformationMapList_ptr->end(), it;
+        //it = transformationMapList_.find(nodeElementName);
+        it = transformationMapList_ptr->find(nodeElementName);
         if (ite != it)
+        {
 …
 } // namespace xios

XIOS/dev/branch_openmp/src/node/axis.hpp

-                      r1205
+                      r1328
        private:
          static bool initializeTransformationMap(std::map<StdString, ETranformationType>& m);
-         //static bool initializeTransformationMap(std::map<StdString, ETranformationType>* m);
          static bool initializeTransformationMap();
          //static std::map<StdString, ETranformationType> transformationMapList_;
          static std::map<StdString, ETranformationType> *transformationMapList_ptr;
+         #pragma omp threadprivate(transformationMapList_ptr)
+         //static bool dummyTransformationMapList_;
+         //#pragma omp threadprivate(dummyTransformationMapList_)
+         static bool dummyTransformationMapList_;
          DECLARE_REF_FUNC(Axis,axis)
 …
 #endif // __XIOS_CAxis__

XIOS/dev/branch_openmp/src/node/compute_connectivity_domain.hpp

r1134	r1328
60	60	static CTransformation<CDomain>* create(const StdString& id, xml::CXMLNode* node);
61	61	static bool _dummyRegistered;
62		~~#pragma omp threadprivate(_dummyRegistered)~~
63
64	62	}; // class CComputeConnectivityDomain
65	63

XIOS/dev/branch_openmp/src/node/context.cpp

-                      r1287
+                      r1328
 #include "context.hpp"
 #include "attribute_template.hpp"
 …
 #include "memtrack.hpp"
+using namespace ep_lib;
 namespace xios {
+  //shared_ptr<CContextGroup> CContext::root;
+  boost::shared_ptr<CContextGroup> * CContext::root_ptr = 0;
+   /// ////////////////////// Dfinitions ////////////////////// ///
+  shared_ptr<CContextGroup> CContext::root;
+   /// ////////////////////// DÃ©finitions ////////////////////// ///
    CContext::CContext(void)
 …
    CContextGroup* CContext::getRoot(void)
+   {
+      //if (root.get()==NULL) root=shared_ptr<CContextGroup>(new CContextGroup(xml::CXMLNode::GetRootName()));
+      //return root.get();
+      //static shared_ptr<CContextGroup> *root_ptr;
+      if(root_ptr == 0) //root_ptr = new shared_ptr<CContextGroup>;
+      // if (root_ptr->get()==NULL)
+      root_ptr = new boost::shared_ptr<CContextGroup>(new CContextGroup(xml::CXMLNode::GetRootName()));
+      return root_ptr->get();
+      if (root.get()==NULL) root=shared_ptr<CContextGroup>(new CContextGroup(xml::CXMLNode::GetRootName()));
+      return root.get();
+   }
 …
       if (!this->hasChild())
+      {
          //oss << "<!-- No definition -->" << std::endl; // fait planter l'incrmentation
+         //oss << "<!-- No definition -->" << std::endl; // fait planter l'incrÃ©mentation
+      }
       else
 …
+   {
      hasClient=true;
+     client = new CContextClient(this, intraComm, interComm, cxtServer);
+     int tmp_rank;
+     MPI_Comm_rank(intraComm, &tmp_rank);
+     MPI_Barrier(intraComm);
+     client = new CContextClient(this,intraComm, interComm, cxtServer);
      registryIn=new CRegistry(intraComm);
      registryIn->setPath(getId()) ;
 …
      else
+     {
        MPI_Comm_dup(intraComm, &intraCommServer);
+       ep_lib::MPI_Comm_dup(intraComm, &intraCommServer);
        comms.push_back(intraCommServer);
        MPI_Comm_dup(interComm, &interCommServer);
+       ep_lib::MPI_Comm_dup(interComm, &interCommServer);
        comms.push_back(interCommServer);
+     }
 …
      else
+     {
        MPI_Comm_dup(intraComm, &intraCommClient);
+       ep_lib::MPI_Comm_dup(intraComm, &intraCommClient);
        comms.push_back(intraCommClient);
        MPI_Comm_dup(interComm, &interCommClient);
+       ep_lib::MPI_Comm_dup(interComm, &interCommClient);
        comms.push_back(interCommClient);
+     }
 …
         for (std::list<ep_lib::MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
           MPI_Comm_free(&(*it));
+          ep_lib::MPI_Comm_free(&(*it));
         comms.clear();
+      }
 …
+   }
    void CContext::checkPrefetchingOfEnabledReadModeFiles()
+   void CContext::doPostTimestepOperationsForEnabledReadModeFiles()
+   {
      int size = enabledReadModeFiles.size();
      for (int i = 0; i < size; ++i)
+     {
         enabledReadModeFiles[i]->prefetchEnabledReadModeFieldsIfNeeded();
+        enabledReadModeFiles[i]->doPostTimestepOperationsForEnabledReadModeFields();
+     }
+   }
 …
+  }
    void CContext::solveAllInheritance(bool apply)
+   {
      // Rsolution des hritages descendants (cd des hritages de groupes)
+   void CContext::solveAllInheritance(bool apply) // default : apply = true
+   {
+     // RÃ©solution des hÃ©ritages descendants (cÃ d des hÃ©ritages de groupes)
      // pour chacun des contextes.
       solveDescInheritance(apply);
      // Rsolution des hritages par rfrence au niveau des fichiers.
+     // RÃ©solution des hÃ©ritages par rÃ©fÃ©rence au niveau des fichiers.
       const vector<CFile*> allFiles=CFile::getAll();
       const vector<CGrid*> allGrids= CGrid::getAll();
 …
       for (unsigned int i = 0; i < allFiles.size(); i++)
          if (!allFiles[i]->enabled.isEmpty()) // Si l'attribut 'enabled' est dfini.
+         if (!allFiles[i]->enabled.isEmpty()) // Si l'attribut 'enabled' est dÃ©fini.
+         {
             if (allFiles[i]->enabled.getValue()) // Si l'attribut 'enabled' est fix  vrai.
+            if (allFiles[i]->enabled.getValue()) // Si l'attribut 'enabled' est fixÃ© Ã  vrai.
+            {
               if ((initDate + allFiles[i]->output_freq.getValue()) < (initDate + this->getCalendar()->getTimeStep()))
 …
       if (enabledFiles.size() == 0)
          DEBUG(<<"Aucun fichier ne va tre sorti dans le contexte nomm \""
+         DEBUG(<<"Aucun fichier ne va Ãªtre sorti dans le contexte nommÃ© \""
                << getId() << "\" !");
+   }
 …
    void CContext::postProcessing()
+   {
-     int myRank;
-     MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
      if (isPostProcessed) return;
 …
       prepareTimeseries();
       //Initialisation du vecteur 'enabledFiles' contenant la liste des fichiers  sortir.
+      //Initialisation du vecteur 'enabledFiles' contenant la liste des fichiers Ã  sortir.
       this->findEnabledFiles();
       this->findEnabledReadModeFiles();
 …
    void CContext::updateCalendar(int step)
+   {
+      #pragma omp critical (_output)
+      {info(50) << "updateCalendar : before : " << calendar->getCurrentDate() << endl;}
+      info(50) << "updateCalendar : before : " << calendar->getCurrentDate() << endl;
       calendar->update(step);
+      #pragma omp critical (_output)
+      {info(50) << "updateCalendar : after : " << calendar->getCurrentDate() << endl;}
+      info(50) << "updateCalendar : after : " << calendar->getCurrentDate() << endl;
 #ifdef XIOS_MEMTRACK_LIGHT
+      #pragma omp critical (_output)
+      {info(50) << " Current memory used by XIOS : "<<  MemTrack::getCurrentMemorySize()*1.0/(1024*1024)<<" Mbyte, at timestep "<<step<<" of context "<<this->getId()<<endl ;}
+      info(50) << " Current memory used by XIOS : "<<  MemTrack::getCurrentMemorySize()*1.0/(1024*1024)<<" Mbyte, at timestep "<<step<<" of context "<<this->getId()<<endl ;
 #endif
       if (hasClient)
+      {
         checkPrefetchingOfEnabledReadModeFiles();
+        doPostTimestepOperationsForEnabledReadModeFiles();
         garbageCollector.invalidate(calendar->getCurrentDate());
+      }
 …
     CContext* context = CObjectFactory::CreateObject<CContext>(id).get();
     getRoot();
     if (!hasctxt) CGroupFactory::AddChild(*root_ptr, context->getShared());
+    if (!hasctxt) CGroupFactory::AddChild(root, context->getShared());
 #define DECLARE_NODE(Name_, name_) \

XIOS/dev/branch_openmp/src/node/context.hpp

-                      r1287
+                      r1328
 #include "xios_spl.hpp"
 //#include "node_type.hpp"
+#include "mpi_std.hpp"
 #include "calendar_wrapper.hpp"
 …
 #include "garbage_collector.hpp"
 #include "registry.hpp"
-#include "mpi.hpp"
 …
          void buildFilterGraphOfEnabledFields();
          void startPrefetchingOfEnabledReadModeFiles();
          void checkPrefetchingOfEnabledReadModeFiles();
+         void doPostTimestepOperationsForEnabledReadModeFiles();
          void findFieldsWithReadAccess(void);
          void solveAllRefOfFieldsWithReadAccess();
 …
          // Context root
+         //static shared_ptr<CContextGroup> root;
+         static boost::shared_ptr<CContextGroup> *root_ptr;
+         #pragma omp threadprivate(root_ptr)
+         static shared_ptr<CContextGroup> root;
          // Determine context on client or not
 …
          // Concrete contex client
          CContextClient* client;
          CRegistry* registryIn ;  //!< input registry which is read from file
          CRegistry* registryOut ; //!< output registry which will be wrote on file at the finalize
       private:

XIOS/dev/branch_openmp/src/node/domain.cpp

-                      r1138
+                      r1328
 namespace xios {
    /// ////////////////////// Dfinitions ////////////////////// ///
+   /// ////////////////////// DÃ©finitions ////////////////////// ///
    CDomain::CDomain(void)
 …
    //std::map<StdString, ETranformationType> CDomain::transformationMapList_ = std::map<StdString, ETranformationType>();
+   std::map<StdString, ETranformationType> *CDomain::transformationMapList_ptr = 0;
    //bool CDomain::_dummyTransformationMapList = CDomain::initializeTransformationMap(CDomain::transformationMapList_);
-   std::map<StdString, ETranformationType> *CDomain::transformationMapList_ptr = 0;
    bool CDomain::initializeTransformationMap(std::map<StdString, ETranformationType>& m)
 …
      (*CDomain::transformationMapList_ptr)["expand_domain"] = TRANS_EXPAND_DOMAIN;
+   }
    const std::set<StdString> & CDomain::getRelFiles(void) const
 …
       break;
+     }
+     completeLonLatClient() ;
+   }
 …
+   {
           CContext* context = CContext::getCurrent();
           CContextClient* client = context->client;
+      CContextClient* client = context->client;
           lon_g.resize(ni_glo) ;
           lat_g.resize(nj_glo) ;
 …
       if (context->hasClient)
       { // Ct client uniquement
+      { // CÃŽtÃ© client uniquement
          this->checkMask();
          this->checkDomainData();
 …
+      }
       else
       { // Ct serveur uniquement
+      { // CÃŽtÃ© serveur uniquement
+      }
 …
       if (context->hasClient)
       { // Ct client uniquement
+      { // CÃŽtÃ© client uniquement
          this->checkMask();
          this->checkDomainData();
 …
+      }
       else
       { // Ct serveur uniquement
+      { // CÃŽtÃ© serveur uniquement
+      }
 …
                                                                                 client->intraComm);
     clientServerMap->computeServerIndexMapping(globalIndexDomain);
     const CClientServerMapping::GlobalIndexMap& globalIndexDomainOnServer = clientServerMap->getGlobalIndexOnServer();
 …
         nodeElementName = node.getElementName();
         if(transformationMapList_ptr == 0) initializeTransformationMap();
+        std::map<StdString, ETranformationType>::const_iterator ite = (*transformationMapList_ptr).end(), it;
+        it = (*transformationMapList_ptr).find(nodeElementName);
+        //std::map<StdString, ETranformationType>::const_iterator ite = transformationMapList_.end(), it;
+        std::map<StdString, ETranformationType>::const_iterator ite = transformationMapList_ptr->end(), it;
+        //it = transformationMapList_.find(nodeElementName);
+        it = transformationMapList_ptr->find(nodeElementName);
         if (ite != it)
+        {

XIOS/dev/branch_openmp/src/node/domain.hpp

-                      r1134
+                      r1328
          static bool initializeTransformationMap();
          //static std::map<StdString, ETranformationType> transformationMapList_;
          static std::map<StdString, ETranformationType> *transformationMapList_ptr;
+         #pragma omp threadprivate(transformationMapList_ptr)
+         //static bool _dummyTransformationMapList;
+         //#pragma omp threadprivate(_dummyTransformationMapList)
+         static bool _dummyTransformationMapList;
          DECLARE_REF_FUNC(Domain,domain)

XIOS/dev/branch_openmp/src/node/expand_domain.hpp

r1134	r1328
60	60	static CTransformation<CDomain>* create(const StdString& id, xml::CXMLNode* node);
61	61	static bool _dummyRegistered;
62		~~#pragma omp threadprivate(_dummyRegistered)~~
63
64	62	}; // class CExpandDomain
65	63

XIOS/dev/branch_openmp/src/node/extract_axis_to_scalar.hpp

r1134	r1328
60	60	static CTransformation<CScalar>* create(const StdString& id, xml::CXMLNode* node);
61	61	static bool _dummyRegistered;
62		~~#pragma omp threadprivate(_dummyRegistered)~~
63	62	}; // class CExtractAxisToScalar
64	63

XIOS/dev/branch_openmp/src/node/extract_domain_to_axis.hpp

r1134	r1328
60	60	static CTransformation<CAxis>* create(const StdString& id, xml::CXMLNode* node);
61	61	static bool _dummyRegistered;
62		~~#pragma omp threadprivate(_dummyRegistered)~~
63	62	}; // class CExtractDomainToAxis
64	63

XIOS/dev/branch_openmp/src/node/field.cpp

-                      r1209
+                      r1328
 #include "spatial_transform_filter.hpp"
+#include <stdio.h>
 namespace xios{
    /// ////////////////////// Dfinitions ////////////////////// ///
+  CField* CField::my_getDirectFieldReference(void) const
+  {
+    // if (this->field_ref.isEmpty())
+    // ERROR("C" #type "* C" #type "::getDirect" #type "Reference(void)",
+    //       << "The " #name_ " with id = '" << getId() << "'"
+    //       << " has no " #name_ "_ref.");
+    // if (!C##type::has(this->name_##_ref))
+    // ERROR("C" #type "* C" #type "::getDirect" #type "Reference(void)",
+    //       << this->name_##_ref
+    //       << " refers to an unknown " #name_ " id.");
+    return CField::get(this->field_ref);
+  }
    CField::CField(void)
 …
       , hasTimeInstant(false)
       , hasTimeCentered(false)
+      , wasDataRequestedFromServer(false)
       , wasDataAlreadyReceivedFromServer(false)
       , isEOF(false)
 …
       , hasTimeInstant(false)
       , hasTimeCentered(false)
+      , wasDataRequestedFromServer(false)
       , wasDataAlreadyReceivedFromServer(false)
       , isEOF(false)
 …
     lastDataRequestedFromServer = tsDataRequested;
+    if (!isEOF) // No need to send the request if we already know we are at EOF
+    // No need to send the request if we are sure that we are already at EOF
+    if (!isEOF || context->getCalendar()->getCurrentDate() <= dateEOF)
+    {
       CEventClient event(getType(), EVENT_ID_READ_DATA);
 …
     else
       serverSourceFilter->signalEndOfStream(tsDataRequested);
+    wasDataRequestedFromServer = true;
     return !isEOF;
 …
     while (currentDate >= lastDataRequestedFromServer)
+    {
+      #pragma omp critical (_output)
+      {
+        info(20) << "currentDate : " << currentDate << endl ;
+        info(20) << "lastDataRequestedFromServer : " << lastDataRequestedFromServer << endl ;
+        info(20) << "file->output_freq.getValue() : " << file->output_freq.getValue() << endl ;
+        info(20) << "lastDataRequestedFromServer + file->output_freq.getValue() : " << lastDataRequestedFromServer + file->output_freq << endl ;
+      }
+      info(20) << "currentDate : " << currentDate << endl ;
+      info(20) << "lastDataRequestedFromServer : " << lastDataRequestedFromServer << endl ;
+      info(20) << "file->output_freq.getValue() : " << file->output_freq.getValue() << endl ;
+      info(20) << "lastDataRequestedFromServer + file->output_freq.getValue() : " << lastDataRequestedFromServer + file->output_freq << endl ;
       dataRequested |= sendReadDataRequest(lastDataRequestedFromServer + file->output_freq);
+    }
 …
+  {
     CContext* context = CContext::getCurrent();
-    int record;
     std::map<int, CArray<double,1> > data;
+    const bool wasEOF = isEOF;
     for (int i = 0; i < ranks.size(); i++)
+    {
       int rank = ranks[i];
+      int record;
       *buffers[i] >> record;
       isEOF = (record == int(-1));
 …
     if (isEOF)
+    {
+      if (!wasEOF)
+        dateEOF = lastDataReceivedFromServer;
       serverSourceFilter->signalEndOfStream(lastDataReceivedFromServer);
+    }
     else
       serverSourceFilter->streamDataFromServer(lastDataReceivedFromServer, data);
+  }
+  void CField::checkForLateDataFromServer(void)
+  {
+    CContext* context = CContext::getCurrent();
+    const CDate& currentDate = context->getCalendar()->getCurrentDate();
+    // Check if data previously requested has been received as expected
+    if (wasDataRequestedFromServer && (!isEOF || currentDate <= dateEOF))
+    {
+      CTimer timer("CField::checkForLateDataFromServer");
+      bool isDataLate;
+      do
+      {
+        const CDate nextDataDue = wasDataAlreadyReceivedFromServer ? (lastDataReceivedFromServer + file->output_freq) : context->getCalendar()->getInitDate();
+        isDataLate = nextDataDue < currentDate;
+        if (isDataLate)
+        {
+          timer.resume();
+          context->checkBuffersAndListen();
+          timer.suspend();
+        }
+      }
+      while (isDataLate && timer.getCumulatedTime() < CXios::recvFieldTimeout);
+      if (isDataLate)
+        ERROR("void CField::checkForLateDataFromServer(void)",
+              << "Late data at timestep = " << currentDate);
+    }
+  }
 …
      CContext* context = CContext::getCurrent();
      solveOnlyReferenceEnabledField(doSending2Server);
+     int myRank;
      MPI_Comm_rank(context->client->intraComm, &myRank);
+     //std::cout<<"Field "<<this->getId()<<" areAllReferenceSolved = "<<areAllReferenceSolved<<std::endl;
      if (!areAllReferenceSolved)
+     {
         areAllReferenceSolved = true;
+        //std::cout<<"Field "<<this->getId()<<" all reference solved"<<std::endl;
         if (context->hasClient)
+        {
 …
        // Check if the data is to be read from a file
        else if (file && !file->mode.isEmpty() && file->mode == CFile::mode_attr::read)
          instantDataFilter = serverSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid,
                                                                                                      freq_offset.isEmpty() ? NoneDu : freq_offset,
                                                                                                      true,
+       {
+         checkAttributes();
+         instantDataFilter = serverSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, freq_offset, true,
                                                                                                      detectMissingValues, defaultValue));
+       }
        else // The data might be passed from the model
+       {
 …
+       {
          if (!serverSourceFilter)
            serverSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid,
                                                                                    freq_offset.isEmpty() ? NoneDu : freq_offset,
                                                                                    true,
+         {
+           checkAttributes();
+           serverSourceFilter = boost::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid, freq_offset, true,
                                                                                    detectMissingValues, defaultValue));
+         }
          selfReferenceFilter = serverSourceFilter;
 …
                << "An operation must be defined for field \"" << getId() << "\".");
+       if (freq_op.isEmpty())
+         freq_op.setValue(TimeStep);
+       if (freq_offset.isEmpty())
+         freq_offset.setValue(NoneDu);
+       checkAttributes();
        const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
        boost::shared_ptr<CTemporalFilter> temporalFilter(new CTemporalFilter(gc, operation,
                                                                              CContext::getCurrent()->getCalendar()->getInitDate(),
 …
                << "An operation must be defined for field \"" << getId() << "\".");
+       if (freq_op.isEmpty()) freq_op.setValue(TimeStep);
+       if (freq_offset.isEmpty()) freq_offset.setValue(NoneDu);
+       checkAttributes();
        const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
        boost::shared_ptr<CTemporalFilter> temporalFilter(new CTemporalFilter(gc, operation,
                                                                              CContext::getCurrent()->getCalendar()->getInitDate(),
 …
    /*!
+    * Check on freq_off and freq_op attributes.
+    */
+   void CField::checkAttributes(void)
+   {
+     bool isFieldRead = file && !file->mode.isEmpty() && file->mode == CFile::mode_attr::read;
+     if (isFieldRead && operation.getValue() != "instant")
+       ERROR("void CField::checkAttributes(void)",
+             << "Unsupported operation for field '" << getFieldOutputName() << "'." << std::endl
+             << "Currently only \"instant\" is supported for fields read from file.")
+     if (freq_op.isEmpty())
+     {
+       if (operation.getValue() == "instant")
+         freq_op.setValue(file->output_freq.getValue());
+       else
+         freq_op.setValue(TimeStep);
+     }
+     if (freq_offset.isEmpty())
+       freq_offset.setValue(isFieldRead ? NoneDu : (freq_op.getValue() - TimeStep));
+   }
+   /*!
     * Returns string arithmetic expression associated to the field.
     * \return if content is defined return content string, otherwise, if "expr" attribute is defined, return expr string.

XIOS/dev/branch_openmp/src/node/field.hpp

-                      r1119
+                      r1328
         static void recvReadDataReady(CEventServer& event);
         void recvReadDataReady(vector<int> ranks, vector<CBufferIn*> buffers);
+        void checkForLateDataFromServer(void);
         void outputField(CArray<double,3>& fieldOut);
         void outputField(CArray<double,2>& fieldOut);
 …
         void sendAddAllVariables();
+        /// VÃ©rifications ///
+        void checkAttributes(void);
         const std::vector<StdString>& getRefDomainAxisIds();
 …
         const string& getExpression(void);
         bool hasExpression(void) const;
+        CField* my_getDirectFieldReference(void) const;
       public:
 …
          bool isEOF;
          CDate lastlast_Write_srv, last_Write_srv, last_operation_srv;
          CDate lastDataRequestedFromServer, lastDataReceivedFromServer;
          bool wasDataAlreadyReceivedFromServer;
+         CDate lastDataRequestedFromServer, lastDataReceivedFromServer, dateEOF;
+         bool wasDataRequestedFromServer, wasDataAlreadyReceivedFromServer;
          map<int,boost::shared_ptr<func::CFunctor> > foperation_srv;

XIOS/dev/branch_openmp/src/node/file.cpp

-                      r1287
+                      r1328
 #include "timer.hpp"
 namespace xios {
 …
    CFile::CFile(void)
       : CObjectTemplate<CFile>(), CFileAttributes()
       , vFieldGroup(), data_out(), enabledFields(), fileComm(MPI_COMM_NULL)
+      , vFieldGroup(), data_out(), enabledFields()
       , allDomainEmpty(false), isOpen(false)
+   {
 …
    CFile::CFile(const StdString & id)
       : CObjectTemplate<CFile>(id), CFileAttributes()
       , vFieldGroup(), data_out(), enabledFields(), fileComm(MPI_COMM_NULL)
+      , vFieldGroup(), data_out(), enabledFields()
       , allDomainEmpty(false), isOpen(false)
+    {
 …
       // create sub communicator for file
       int color = allDomainEmpty ? 0 : 1;
       MPI_Comm_split(server->intraComm, color, server->intraCommRank, &fileComm);
       if (allDomainEmpty) MPI_Comm_free(&fileComm);
+      ep_lib::MPI_Comm_split(server->intraComm, color, server->intraCommRank, &fileComm);
+      if (allDomainEmpty) ep_lib::MPI_Comm_free(&fileComm);
       // if (time_counter.isEmpty()) time_counter.setValue(time_counter_attr::centered);
 …
+         {
             int commSize, commRank;
             MPI_Comm_size(fileComm, &commSize);
             MPI_Comm_rank(fileComm, &commRank);
+            ep_lib::MPI_Comm_size(fileComm, &commSize);
+            ep_lib::MPI_Comm_rank(fileComm, &commRank);
             if (server->intraCommSize > 1)
 …
          if (isOpen) data_out->closeFile();
         data_out = boost::shared_ptr<CDataOutput>(new CNc4DataOutput(this, oss.str(), append, useClassicFormat, useCFConvention,
                                                               static_cast< ::MPI_Comm >(fileComm.mpi_comm), multifile, isCollective, time_counter_name));
+        data_out = shared_ptr<CDataOutput>(new CNc4DataOutput(this, oss.str(), append, useClassicFormat, useCFConvention,
+                                                              fileComm, multifile, isCollective, time_counter_name));
         isOpen = true;
 …
+      {
         int commSize, commRank;
         MPI_Comm_size(fileComm, &commSize);
         MPI_Comm_rank(fileComm, &commRank);
+        ep_lib::MPI_Comm_size(fileComm, &commSize);
+        ep_lib::MPI_Comm_rank(fileComm, &commRank);
         if (server->intraCommSize > 1)
 …
       bool isCollective = par_access.isEmpty() || par_access == par_access_attr::collective;
-      #ifdef _usingEP
-      //printf("multifile was %d\n", multifile);
-      multifile = true;
       if (isOpen) data_out->closeFile();
       if (time_counter_name.isEmpty()) data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), static_cast< ::MPI_Comm >(fileComm.mpi_comm), multifile, isCollective));
       else data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), static_cast< ::MPI_Comm >(fileComm.mpi_comm), multifile, isCollective, time_counter_name));
+      if (time_counter_name.isEmpty()) data_in = shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), fileComm, multifile, isCollective));
+      else data_in = shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), fileComm, multifile, isCollective, time_counter_name));
       isOpen = true;
-      #else
-      if (isOpen) data_out->closeFile();
-      if (time_counter_name.isEmpty()) data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), static_cast< ::MPI_Comm >(fileComm.mpi_comm), multifile, isCollective));
-      else data_in = boost::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), static_cast< ::MPI_Comm >(fileComm.mpi_comm), multifile, isCollective, time_counter_name));
-      isOpen = true;
-      #endif
+    }
+  }
 …
           this->data_in->closeFile();
+       }
+      if (fileComm != MPI_COMM_NULL) MPI_Comm_free(&fileComm);
+//      if (fileComm != MPI_COMM_NULL) MPI_Comm_free(&fileComm);
+      //if (fileComm.mpi_comm != ::MPI_COMM_NULL) MPI_Comm_free(&fileComm);
+   }
    //----------------------------------------------------------------
 …
      // It would probably be better to call initFile() somehow
+     MPI_Comm_dup(client->intraComm, &fileComm);
+     ep_lib::MPI_Comm_dup(client->intraComm, &fileComm);
      if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
+     //#pragma omp critical (_readAttributesOfEnabledFieldsInReadMode_)
+     //{
+     checkFile(); // calls nc_open
+     checkFile();
      for (int idx = 0; idx < enabledFields.size(); ++idx)
 …
         // Read necessary value from file
+        #pragma omp critical (_func)
+        {
+          //checkFile();
+          this->data_in->readFieldAttributesValues(enabledFields[idx]);
+          //close();
+        }
+        this->data_in->readFieldAttributesValues(enabledFields[idx]);
         // Fill attributes for base reference
         enabledFields[idx]->solveGridDomainAxisBaseRef();
 …
      // Now everything is ok, close it
      close();
-     //}
-     //if (fileComm != MPI_COMM_NULL) MPI_Comm_free(&fileComm);
+   }
 …
    /*!
+     Prefetching the data for enabled fields read from file whose data is out-of-date.
+   */
+   void CFile::prefetchEnabledReadModeFieldsIfNeeded(void)
+     Do all post timestep operations for enabled fields in read mode:
+      - Prefetch the data read from file when needed
+      - Check that the data excepted from server has been received
+   */
+   void CFile::doPostTimestepOperationsForEnabledReadModeFields(void)
+   {
      if (mode.isEmpty() || mode.getValue() != mode_attr::read)
 …
      int size = this->enabledFields.size();
      for (int i = 0; i < size; ++i)
+     {
+       this->enabledFields[i]->checkForLateDataFromServer();
        this->enabledFields[i]->sendReadDataRequestIfNeeded();
+     }
+   }
 …
        CField* field = this->enabledFields[i];
        this->sendAddField(field->getId());
+       field->checkAttributes();
        field->sendAllAttributesToServer();
        field->sendAddAllVariables();

XIOS/dev/branch_openmp/src/node/file.hpp

-                      r1134
+                      r1328
 /// XIOS headers ///
 #include "xios_spl.hpp"
+#include "mpi_std.hpp"
 #include "field.hpp"
 #include "data_output.hpp"
 …
 #include "attribute_enum.hpp"
 #include "attribute_enum_impl.hpp"
-#include "mpi.hpp"
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 namespace xios {
 …
          void buildFilterGraphOfEnabledFields(CGarbageCollector& gc);
          void prefetchEnabledReadModeFields();
          void prefetchEnabledReadModeFieldsIfNeeded();
+         void doPostTimestepOperationsForEnabledReadModeFields();
          // Add component into file

XIOS/dev/branch_openmp/src/node/generate_rectilinear_domain.hpp

r1134	r1328
60	60	static CTransformation<CDomain>* create(const StdString& id, xml::CXMLNode* node);
61	61	static bool _dummyRegistered;
62		~~#pragma omp threadprivate(_dummyRegistered)~~
63	62	}; // class CGenerateRectilinearDomain
64	63

XIOS/dev/branch_openmp/src/node/grid.cpp

r1205	r1328
1114	1114	outLocalIndexToServer(idx) = itIndex->second;
1115	1115	}
1116
	1116
1117	1117	const std::list<int>& ranks = client->getRanksServerLeader();
1118	1118	for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
…	…
1750	1750	pDom->solveRefInheritance(apply);
1751	1751	pDom->solveInheritanceTransformation();
	1752	//cout<<"pDom check"<<endl;
1752	1753	}
1753	1754	}

XIOS/dev/branch_openmp/src/node/interpolate_axis.hpp

r1134	r1328
62	62	static CTransformation<CAxis>* create(const StdString& id, xml::CXMLNode* node);
63	63	static bool _dummyRegistered;
64		~~#pragma omp threadprivate(_dummyRegistered)~~
65
66	64	}; // class CInterpolateAxis
67	65

XIOS/dev/branch_openmp/src/node/interpolate_domain.hpp

r1134	r1328
60	60	static CTransformation<CDomain>* create(const StdString& id, xml::CXMLNode* node);
61	61	static bool _dummyRegistered;
62		~~#pragma omp threadprivate(_dummyRegistered)~~
63	62	}; // class CInterpolateDomain
64	63

XIOS/dev/branch_openmp/src/node/inverse_axis.hpp

r1134	r1328
59	59	static CTransformation<CAxis>* create(const StdString& id, xml::CXMLNode* node);
60	60	static bool _dummyRegistered;
61		~~#pragma omp threadprivate(_dummyRegistered)~~
62	61
63	62	}; // class CInverseAxis

XIOS/dev/branch_openmp/src/node/mesh.cpp

-                      r1134
+                      r1328
 #include "mesh.hpp"
+using namespace ep_lib;
 namespace xios {
 …
+  }
   std::map <StdString, CMesh> CMesh::meshList = std::map <StdString, CMesh>();
   std::map <StdString, vector<int> > CMesh::domainList = std::map <StdString, vector<int> >();
+  //std::map <StdString, CMesh> CMesh::meshList = std::map <StdString, CMesh>();
+  //std::map <StdString, vector<int> > CMesh::domainList = std::map <StdString, vector<int> >();
   std::map <StdString, CMesh> *CMesh::meshList_ptr = 0;
   std::map <StdString, vector<int> > *CMesh::domainList_ptr = 0;
 ///---------------------------------------------------------------
 …
  * \param [in] nvertex Number of verteces (1 for nodes, 2 for edges, 3 and up for faces).
  */
-/* bkp
-  CMesh* CMesh::getMesh (StdString meshName, int nvertex)
+  {
-    CMesh::domainList[meshName].push_back(nvertex);
-    if ( CMesh::meshList.begin() != CMesh::meshList.end() )
+    {
-      for (std::map<StdString, CMesh>::iterator it=CMesh::meshList.begin(); it!=CMesh::meshList.end(); ++it)
+      {
-        if (it->first == meshName)
-          return &meshList[meshName];
-        else
+        {
-          CMesh newMesh;
-          CMesh::meshList.insert( make_pair(meshName, newMesh) );
-          return &meshList[meshName];
+        }
+      }
+    }
-    else
+    {
-      CMesh newMesh;
-      CMesh::meshList.insert( make_pair(meshName, newMesh) );
-      return &meshList[meshName];
+    }
+  }
-*/
   CMesh* CMesh::getMesh (StdString meshName, int nvertex)
+  {
 …
     if(CMesh::meshList_ptr == NULL)   CMesh::meshList_ptr   = new std::map <StdString, CMesh>();
+    (*CMesh::domainList_ptr)[meshName].push_back(nvertex);
+    if ( (*CMesh::meshList_ptr).begin() != (*CMesh::meshList_ptr).end() )
+    {
+      for (std::map<StdString, CMesh>::iterator it=(*CMesh::meshList_ptr).begin(); it!=(*CMesh::meshList_ptr).end(); ++it)
+    //CMesh::domainList[meshName].push_back(nvertex);
+    CMesh::domainList_ptr->at(meshName).push_back(nvertex);
+    //if ( CMesh::meshList.begin() != CMesh::meshList.end() )
+    if ( CMesh::meshList_ptr->begin() != CMesh::meshList_ptr->end() )
+    {
+      //for (std::map<StdString, CMesh>::iterator it=CMesh::meshList.begin(); it!=CMesh::meshList.end(); ++it)
+      for (std::map<StdString, CMesh>::iterator it=CMesh::meshList_ptr->begin(); it!=CMesh::meshList_ptr->end(); ++it)
+      {
         if (it->first == meshName)
+          return &((*CMesh::meshList_ptr)[meshName]);
+          //return &meshList[meshName];
+          return &meshList_ptr->at(meshName);
         else
+        {
           CMesh newMesh;
+          (*CMesh::meshList_ptr).insert( make_pair(meshName, newMesh) );
+          return &((*CMesh::meshList_ptr)[meshName]);
+          //CMesh::meshList.insert( make_pair(meshName, newMesh) );
+          CMesh::meshList_ptr->insert( make_pair(meshName, newMesh) );
+          //return &meshList[meshName];
+          return &meshList_ptr->at(meshName);
+        }
+      }
 …
+    {
       CMesh newMesh;
+      (*CMesh::meshList_ptr).insert( make_pair(meshName, newMesh) );
+      return &((*CMesh::meshList_ptr)[meshName]);
+      //CMesh::meshList.insert( make_pair(meshName, newMesh) );
+      CMesh::meshList_ptr->insert( make_pair(meshName, newMesh) );
+      //return &meshList[meshName];
+      return &meshList_ptr->at(meshName);
+    }
+  }
 …
  * \param [in] bounds_lat Array of boundary latitudes. Its size depends on the element type.
  */
   void CMesh::createMeshEpsilon(const ep_lib::MPI_Comm& comm,
+  void CMesh::createMeshEpsilon(const MPI_Comm& comm,
                                 const CArray<double, 1>& lonvalue, const CArray<double, 1>& latvalue,
                                 const CArray<double, 2>& bounds_lon, const CArray<double, 2>& bounds_lat)
 …
    */
   void CMesh::getGloNghbFacesNodeType(const ep_lib::MPI_Comm& comm, const CArray<int, 1>& face_idx,
+  void CMesh::getGloNghbFacesNodeType(const MPI_Comm& comm, const CArray<int, 1>& face_idx,
                                const CArray<double, 2>& bounds_lon, const CArray<double, 2>& bounds_lat,
                                CArray<int, 2>& nghbFaces)
 …
    */
   void CMesh::getGloNghbFacesEdgeType(const ep_lib::MPI_Comm& comm, const CArray<int, 1>& face_idx,
+  void CMesh::getGloNghbFacesEdgeType(const MPI_Comm& comm, const CArray<int, 1>& face_idx,
                                const CArray<double, 2>& bounds_lon, const CArray<double, 2>& bounds_lat,
                                CArray<int, 2>& nghbFaces)
 …
    */
   void CMesh::getGlobalNghbFaces(const int nghbType, const ep_lib::MPI_Comm& comm,
+  void CMesh::getGlobalNghbFaces(const int nghbType, const MPI_Comm& comm,
                                  const CArray<int, 1>& face_idx,
                                  const CArray<double, 2>& bounds_lon, const CArray<double, 2>& bounds_lat,

XIOS/dev/branch_openmp/src/node/mesh.hpp

-                      r1134
+                      r1328
       int nbFaces_;
+      static std::map <StdString, CMesh> meshList;
+      static std::map <StdString, vector<int> > domainList;
+      //static std::map <StdString, CMesh> meshList;
       static std::map <StdString, CMesh> *meshList_ptr;
+      //static std::map <StdString, vector<int> > domainList;
       static std::map <StdString, vector<int> > *domainList_ptr;
-      #pragma omp threadprivate(meshList_ptr, domainList_ptr)
       CClientClientDHTSizet* pNodeGlobalIndex;                    // pointer to a map <nodeHash, nodeIdxGlo>
       CClientClientDHTSizet* pEdgeGlobalIndex;                    // pointer to a map <edgeHash, edgeIdxGlo>

XIOS/dev/branch_openmp/src/node/reduce_axis_to_scalar.hpp

r1134	r1328
59	59	static CTransformation<CScalar>* create(const StdString& id, xml::CXMLNode* node);
60	60	static bool _dummyRegistered;
61		~~#pragma omp threadprivate(_dummyRegistered)~~
62	61	}; // class CReduceAxisToScalar
63	62

XIOS/dev/branch_openmp/src/node/reduce_domain_to_axis.hpp

r1134	r1328
60	60	static CTransformation<CAxis>* create(const StdString& id, xml::CXMLNode* node);
61	61	static bool _dummyRegistered;
62		~~#pragma omp threadprivate(_dummyRegistered)~~
63	62	}; // class CReduceDomainToAxis
64	63

XIOS/dev/branch_openmp/src/node/reduce_domain_to_scalar.hpp

r1134	r1328
60	60	static CTransformation<CScalar>* create(const StdString& id, xml::CXMLNode* node);
61	61	static bool _dummyRegistered;
62		~~#pragma omp threadprivate(_dummyRegistered)~~
63	62	}; // class CReduceDomainToScalar
64	63

XIOS/dev/branch_openmp/src/node/scalar.cpp

-                      r1134
+                      r1328
    //std::map<StdString, ETranformationType> CScalar::transformationMapList_ = std::map<StdString, ETranformationType>();
+   std::map<StdString, ETranformationType> *CScalar::transformationMapList_ptr = 0;
    //bool CScalar::dummyTransformationMapList_ = CScalar::initializeTransformationMap(CScalar::transformationMapList_);
-   std::map<StdString, ETranformationType> *CScalar::transformationMapList_ptr = 0;
    bool CScalar::initializeTransformationMap(std::map<StdString, ETranformationType>& m)
+   {
 …
         nodeElementName = node.getElementName();
+        //std::map<StdString, ETranformationType>::const_iterator ite = transformationMapList_.end(), it;
         if(CScalar::transformationMapList_ptr == 0) initializeTransformationMap();
+        std::map<StdString, ETranformationType>::const_iterator ite = (*CScalar::transformationMapList_ptr).end(), it;
+        it = (*CScalar::transformationMapList_ptr).find(nodeElementName);
+        std::map<StdString, ETranformationType>::const_iterator ite = transformationMapList_ptr->end(), it;
+        //it = transformationMapList_.find(nodeElementName);
+        it = transformationMapList_ptr->find(nodeElementName);
         if (ite != it)
+        {

XIOS/dev/branch_openmp/src/node/scalar.hpp

-                      r1134
+                      r1328
            static bool initializeTransformationMap(std::map<StdString, ETranformationType>& m);
            static bool initializeTransformationMap();
+           //static bool dummyTransformationMapList_;
+           //static std::map<StdString, ETranformationType> transformationMapList_;
            static std::map<StdString, ETranformationType> *transformationMapList_ptr;
            #pragma omp threadprivate(transformationMapList_ptr)
+           static bool dummyTransformationMapList_;

XIOS/dev/branch_openmp/src/node/transformation.hpp

-                      r1138
+                      r1328
       typedef std::map<ETranformationType, CreateTransformationCallBack> CallBackMap;
       static CallBackMap* transformationCreationCallBacks_;
-      //#pragma omp threadprivate(transformationCreationCallBacks_)
       static bool registerTransformation(ETranformationType transType, CreateTransformationCallBack createFn);
 …
     if (0 == transformationCreationCallBacks_)
       transformationCreationCallBacks_ = new CallBackMap();
     return (*transformationCreationCallBacks_).insert(make_pair(transType, createFn)).second;
+  }

XIOS/dev/branch_openmp/src/node/zoom_axis.hpp

r1134	r1328
60	60	static CTransformation<CAxis>* create(const StdString& id, xml::CXMLNode* node);
61	61	static bool _dummyRegistered;
62		~~#pragma omp threadprivate(_dummyRegistered)~~
63	62	}; // class CZoomAxis
64	63

XIOS/dev/branch_openmp/src/node/zoom_domain.hpp

r1134	r1328
60	60	static CTransformation<CDomain>* create(const StdString& id, xml::CXMLNode* node);
61	61	static bool _dummyRegistered;
62		~~#pragma omp threadprivate(_dummyRegistered)~~
63
64	62	}; // class CZoomDomain
65	63

XIOS/dev/branch_openmp/src/object_factory.cpp

-                      r1134
+                      r1328
    /// ////////////////////// DÃ©finitions ////////////////////// ///
    StdString *CObjectFactory::CurrContext_ptr = new StdString;
+   StdString CObjectFactory::CurrContext("");
    void CObjectFactory::SetCurrentContextId(const StdString & context)
+   {
+     if(CObjectFactory::CurrContext_ptr == NULL ) CObjectFactory::CurrContext_ptr = new StdString;
+     CObjectFactory::CurrContext_ptr->assign(context);
+   }
+   { CObjectFactory::CurrContext = context; }
    StdString & CObjectFactory::GetCurrentContextId(void)
+   {
+     return (*CObjectFactory::CurrContext_ptr);
+   }
+   { return (CObjectFactory::CurrContext); }
 } // namespace xios

XIOS/dev/branch_openmp/src/object_factory.hpp

-                      r1134
+                      r1328
          /// PropriÃ©tÃ©s statiques ///
+         static StdString *CurrContext_ptr;
+         #pragma omp threadprivate(CurrContext_ptr)
+         static StdString CurrContext;
    }; // class CObjectFactory

XIOS/dev/branch_openmp/src/object_factory_decl.cpp

-                      r1287
+                      r1328
+{
 #define macro(U) \
   template boost::shared_ptr<U> CObjectFactory::GetObject<U>(const StdString& id);  \
   template boost::shared_ptr<U> CObjectFactory::GetObject<U>(const StdString& context,const StdString& id); \
   template boost::shared_ptr<U> CObjectFactory::GetObject<U>(const U* const object); \
+  template shared_ptr<U> CObjectFactory::GetObject<U>(const StdString& id);  \
+  template shared_ptr<U> CObjectFactory::GetObject<U>(const StdString& context,const StdString& id); \
+  template shared_ptr<U> CObjectFactory::GetObject<U>(const U* const object); \
   template int CObjectFactory::GetObjectNum<U>(void); \
   template int CObjectFactory::GetObjectIdNum<U>(void); \
   template const std::vector<boost::shared_ptr<U> >& CObjectFactory::GetObjectVector<U>(const StdString& context ); \
+  template const std::vector<shared_ptr<U> >& CObjectFactory::GetObjectVector<U>(const StdString& context ); \
   template bool CObjectFactory::HasObject<U>(const StdString& id); \
   template bool CObjectFactory::HasObject<U>(const StdString& context,const StdString& id); \

XIOS/dev/branch_openmp/src/object_factory_impl.hpp

-                      r1134
+                      r1328
 namespace xios
+{
    /// ////////////////////// DÃ©finitions ////////////////////// ///
+   /// ////////////////////// Dfinitions ////////////////////// ///
    template <typename U>
        int CObjectFactory::GetObjectNum(void)
+   {
       if (CurrContext_ptr->size() == 0)
+      if (CurrContext.size() == 0)
          ERROR("CObjectFactory::GetObjectNum(void)",
                << "please define current context id !");
+      if(U::AllVectObj == NULL) return 0;
+      return (*U::AllVectObj)[*CObjectFactory::CurrContext_ptr].size();
+      //return (U::AllVectObj[CObjectFactory::CurrContext].size());
+      if(U::AllVectObj_ptr == NULL) return 0;
+      return (*U::AllVectObj_ptr)[CObjectFactory::CurrContext].size();
+   }
 …
       int CObjectFactory::GetObjectIdNum(void)
+   {
       if (CurrContext_ptr->size() == 0)
+      if (CurrContext.size() == 0)
          ERROR("CObjectFactory::GetObjectIdNum(void)",
                << "please define current context id !");
+      if(U::AllMapObj  == NULL) return 0;
+      return (* U::AllMapObj) [*CObjectFactory::CurrContext_ptr].size();
+      //return (U::AllMapObj[CObjectFactory::CurrContext].size());
+      if(U::AllMapObj_ptr == NULL) return 0;
+      return (*U::AllMapObj_ptr)[CObjectFactory::CurrContext].size();
+   }
 …
       bool CObjectFactory::HasObject(const StdString & id)
+   {
       if (CurrContext_ptr->size() == 0)
+      if (CurrContext.size() == 0)
          ERROR("CObjectFactory::HasObject(const StdString & id)",
                << "[ id = " << id << " ] please define current context id !");
+      if(U::AllMapObj  == NULL)  return false;
+      //return (U::AllMapObj[CObjectFactory::CurrContext].find(id) !=
+      //        U::AllMapObj[CObjectFactory::CurrContext].end());
+      if(U::AllMapObj_ptr  == NULL)  return false;
+      return ((*U::AllMapObj_ptr)[CObjectFactory::CurrContext].find(id) !=
+              (*U::AllMapObj_ptr)[CObjectFactory::CurrContext].end());
-      return ((*U::AllMapObj)[*CObjectFactory::CurrContext_ptr].find(id) !=
-              (*U::AllMapObj)[*CObjectFactory::CurrContext_ptr].end());
+   }
 …
       bool CObjectFactory::HasObject(const StdString & context, const StdString & id)
+   {
-      if(U::AllMapObj  == NULL) return false;
+      if (U::AllMapObj->find(context) == U::AllMapObj->end()) return false ;
+      // if (U::AllMapObj.find(context) == U::AllMapObj.end()) return false ;
+      // else return (U::AllMapObj[context].find(id) !=  U::AllMapObj[context].end());
+      if(U::AllMapObj_ptr  == NULL) return false;
+      else
+      {
+         return ((*U::AllMapObj)[context].find(id) !=  (*U::AllMapObj)[context].end());
+      }
+      if (U::AllMapObj_ptr->find(context) == U::AllMapObj_ptr->end()) return false ;
+      else return ((*U::AllMapObj_ptr)[context].find(id) !=  (*U::AllMapObj_ptr)[context].end());
+   }
 …
       boost::shared_ptr<U> CObjectFactory::GetObject(const U * const object)
+   {
+      if(U::AllVectObj == NULL) return (boost::shared_ptr<U>());
+      if (CurrContext_ptr->size() == 0)
+      if(U::AllVectObj_ptr == NULL) return (boost::shared_ptr<U>());
+      if (CurrContext.size() == 0)
          ERROR("CObjectFactory::GetObject(const U * const object)",
                << "please define current context id !");
       std::vector<boost::shared_ptr<U> > & vect =
                      (*U::AllVectObj)[*CObjectFactory::CurrContext_ptr];
+      //std::vector<boost::shared_ptr<U> > & vect = U::AllVectObj[CObjectFactory::CurrContext];
+      std::vector<boost::shared_ptr<U> > & vect = (*U::AllVectObj_ptr)[CObjectFactory::CurrContext];
       typename std::vector<boost::shared_ptr<U> >::const_iterator
 …
       boost::shared_ptr<U> CObjectFactory::GetObject(const StdString & id)
+   {
+      if(U::AllMapObj  == NULL) return (boost::shared_ptr<U>());
+      if (CurrContext_ptr->size() == 0)
+      if(U::AllMapObj_ptr  == NULL) return (boost::shared_ptr<U>());
+      if (CurrContext.size() == 0)
          ERROR("CObjectFactory::GetObject(const StdString & id)",
                << "[ id = " << id << " ] please define current context id !");
 …
                << "[ id = " << id << ", U = " << U::GetName() << " ] "
                << "object was not found.");
+      return (*U::AllMapObj)[*CObjectFactory::CurrContext_ptr][id];
+      //cout<<"CObjectFactory::GetObject(const StdString & id)[ id = " << id << ", U = " << U::GetName() << " ] "<<endl;
+      if(id == "src_domain_regular_read")
+      {
+         //cout<<"match"<<endl;
+      }
+      //return (U::AllMapObj[CObjectFactory::CurrContext][id]);
+      return (*U::AllMapObj_ptr)[CObjectFactory::CurrContext][id];
+   }
 …
       boost::shared_ptr<U> CObjectFactory::GetObject(const StdString & context, const StdString & id)
+   {
       if(U::AllMapObj  == NULL) return (boost::shared_ptr<U>());
+      if(U::AllMapObj_ptr  == NULL) return (boost::shared_ptr<U>());
       if (!CObjectFactory::HasObject<U>(context,id))
 …
                << "object was not found.");
+      return (*U::AllMapObj)[context][id];
+      //cout<<"CObjectFactory::GetObject(const StdString & context, const StdString & id)[ id = " << id << ", U = " << U::GetName() << " ] "<<endl;
+      if(id == "src_domain_regular_read")
+      {
+         //cout<<"match"<<endl;
+         boost::shared_ptr<U> value;
+      }
+      //return (U::AllMapObj[context][id]);
+      return (*U::AllMapObj_ptr)[context][id];
+   }
 …
    boost::shared_ptr<U> CObjectFactory::CreateObject(const StdString& id)
+   {
       if(U::AllVectObj == NULL) U::AllVectObj = new xios_map<StdString, std::vector<boost::shared_ptr<U> > >;
       if(U::AllMapObj  == NULL) U::AllMapObj  = new xios_map<StdString, xios_map<StdString, boost::shared_ptr<U> > >;
+      if(U::AllVectObj_ptr == NULL) U::AllVectObj_ptr = new xios_map<StdString, std::vector<boost::shared_ptr<U> > >;
+      if(U::AllMapObj_ptr  == NULL) U::AllMapObj_ptr  = new xios_map<StdString, xios_map<StdString, boost::shared_ptr<U> > >;
+      if (CurrContext_ptr->empty())
+      if (CurrContext.empty())
          ERROR("CObjectFactory::CreateObject(const StdString& id)",
                << "[ id = " << id << " ] please define current context id !");
 …
          boost::shared_ptr<U> value(new U(id.empty() ? CObjectFactory::GenUId<U>() : id));
+         (* U::AllVectObj)[*CObjectFactory::CurrContext_ptr].insert((*U::AllVectObj)[*CObjectFactory::CurrContext_ptr].end(), value);
+         (* U::AllMapObj) [*CObjectFactory::CurrContext_ptr].insert(std::make_pair(value->getId(), value));
+         //U::AllVectObj[CObjectFactory::CurrContext].insert(U::AllVectObj[CObjectFactory::CurrContext].end(), value);
+         //U::AllMapObj[CObjectFactory::CurrContext].insert(std::make_pair(value->getId(), value));
+         (*U::AllVectObj_ptr)[CObjectFactory::CurrContext].insert((*U::AllVectObj_ptr)[CObjectFactory::CurrContext].end(), value);
+         (*U::AllMapObj_ptr) [CObjectFactory::CurrContext].insert(std::make_pair(value->getId(), value));
+         //cout<<"CObjectFactory::CreateObject(const StdString& id) [ id = " << id << " ]" <<endl;
          return value;
 …
          CObjectFactory::GetObjectVector(const StdString & context)
+   {
+      if(U::AllVectObj != NULL)
+      return (*U::AllVectObj)[context];
+      //return (U::AllVectObj[context]);
+      return (*U::AllVectObj_ptr)[context];
+   }
 …
+   {
       StdOStringStream oss;
       if(U::GenId == NULL) U::GenId = new xios_map< StdString, long int >;
       oss << GetUIdBase<U>() << (*U::GenId)[*CObjectFactory::CurrContext_ptr]++;
+      if(U::GenId_ptr == NULL) U::GenId_ptr = new xios_map< StdString, long int >;
+      oss << GetUIdBase<U>() << (*U::GenId_ptr)[CObjectFactory::CurrContext]++;
       return oss.str();
+   }

XIOS/dev/branch_openmp/src/object_template.hpp

-                      r1287
+                      r1328
          static T* get(const string& contextId, const string& id) ;
          T* get(void) ;
          boost::shared_ptr<T> getShared(void) ;
          static boost::shared_ptr<T> getShared(const T* ptr) ;
+         shared_ptr<T> getShared(void) ;
+         static shared_ptr<T> getShared(const T* ptr) ;
          static T* create(const string& id=string("")) ;
 …
          /// PropriÃ©tÃ©s statiques ///
+         // bkp
+         // static xios_map<StdString,
+         //        xios_map<StdString,
+         //        boost::shared_ptr<DerivedType> > > AllMapObj;
+         // static xios_map<StdString,
+         //        std::vector<boost::shared_ptr<DerivedType> > > AllVectObj;
+         static xios_map<StdString,
+                xios_map<StdString,
+                boost::shared_ptr<DerivedType> > > *AllMapObj_ptr;
+         static xios_map<StdString,
+                std::vector<boost::shared_ptr<DerivedType> > > *AllVectObj_ptr;
+         // static xios_map< StdString, long int > GenId ;
+         static xios_map<StdString, xios_map<StdString, boost::shared_ptr<DerivedType> > > *AllMapObj;
+         static xios_map<StdString, std::vector<boost::shared_ptr<DerivedType> > > *AllVectObj;
+         static xios_map< StdString, long int > *GenId;
+         #pragma omp threadprivate(AllMapObj, AllVectObj, GenId)
+         static xios_map< StdString, long int > *GenId_ptr ;
    }; // class CObjectTemplate

XIOS/dev/branch_openmp/src/object_template_impl.hpp

-                      r1287
+                      r1328
       xios_map<StdString,
       xios_map<StdString,
+      boost::shared_ptr<T> > > *CObjectTemplate<T>::AllMapObj = 0;
+      //boost::shared_ptr<T> > > CObjectTemplate<T>::AllMapObj;
+      boost::shared_ptr<T> > > *CObjectTemplate<T>::AllMapObj_ptr = 0;
    template <class T>
       xios_map<StdString,
+      std::vector<boost::shared_ptr<T> > > *CObjectTemplate<T>::AllVectObj = 0;
+   template <class T>
+      xios_map<StdString,long int> *CObjectTemplate<T>::GenId = 0;
+      //std::vector<boost::shared_ptr<T> > > CObjectTemplate<T>::AllVectObj;
+      std::vector<boost::shared_ptr<T> > > *CObjectTemplate<T>::AllVectObj_ptr = 0;
+   template <class T>
+      //xios_map<StdString,long int> CObjectTemplate<T>::GenId;
+      xios_map<StdString,long int> *CObjectTemplate<T>::GenId_ptr = 0;
    template <class T>
 …
          CObjectTemplate<T>::GetAllVectobject(const StdString & contextId)
+   {
+      return (CObjectTemplate<T>::AllVectObj->at(contextId));
+      //return (CObjectTemplate<T>::AllVectObj[contextId]);
+    return (CObjectTemplate<T>::AllVectObj_ptr->at(contextId));
+   }
 …
    template <typename T>
    boost::shared_ptr<T> CObjectTemplate<T>::getShared(const T* ptr)
+   shared_ptr<T> CObjectTemplate<T>::getShared(const T* ptr)
+   {
      return CObjectFactory::GetObject<T>(ptr);
 …
    template <typename T>
    boost::shared_ptr<T> CObjectTemplate<T>::getShared(void)
+   shared_ptr<T> CObjectTemplate<T>::getShared(void)
+   {
      return CObjectFactory::GetObject<T>((T*)this);
 …
    const vector<T*> CObjectTemplate<T>::getAll()
+   {
      const vector< boost::shared_ptr<T> >& shared_vect= CObjectFactory::GetObjectVector<T>();
+     const vector< shared_ptr<T> >& shared_vect= CObjectFactory::GetObjectVector<T>();
      vector<T*> vect;
      typename vector<boost::shared_ptr<T> >::const_iterator it;
+     typename vector<shared_ptr<T> >::const_iterator it;
      for(it=shared_vect.begin();it!=shared_vect.end();++it) vect.push_back(it->get());
      return vect;
 …
    const vector<T*> CObjectTemplate<T>::getAll(const string & id)
+   {
      const vector< boost::shared_ptr<T> >& shared_vect= CObjectFactory::GetObjectVector<T>(id);
+     const vector< shared_ptr<T> >& shared_vect= CObjectFactory::GetObjectVector<T>(id);
      vector<T*> vect;
      typename vector<boost::shared_ptr<T> >::const_iterator it;
+     typename vector<shared_ptr<T> >::const_iterator it;
      for(it=shared_vect.begin();it!=shared_vect.end();++it) vect.push_back(it->get());
      return vect;

XIOS/dev/branch_openmp/src/parse_expr/lex_parser.cpp

r1134	r1328
347	347	extern char *yytext;
348	348	#define yytext_ptr yytext
349
350	349	static yyconst flex_int16_t yy_nxt[][128] =
351	350	{

XIOS/dev/branch_openmp/src/parse_expr/yacc_parser.cpp

-                      r1134
+                      r1328
+}
+  static IFilterExprNode* parsed;
+  static std::string globalInputText;
+  static std::string *globalInputText_ptr = 0;
+  static size_t globalReadOffset = 0;
+  #pragma omp threadprivate(parsed, globalInputText_ptr, globalReadOffset)
+  IFilterExprNode* parsed;
+  std::string globalInputText;
+  size_t globalReadOffset = 0;
   int readInputForLexer(char* buffer, size_t* numBytesRead, size_t maxBytesToRead)
+  {
-    if(globalInputText_ptr == 0) globalInputText_ptr = new std::string;
     size_t numBytesToRead = maxBytesToRead;
     size_t bytesRemaining = (*globalInputText_ptr).length()-globalReadOffset;
+    size_t bytesRemaining = globalInputText.length()-globalReadOffset;
     size_t i;
     if (numBytesToRead > bytesRemaining) numBytesToRead = bytesRemaining;
     for (i = 0; i < numBytesToRead; i++) buffer[i] = (*globalInputText_ptr).c_str()[globalReadOffset + i];
+    for (i = 0; i < numBytesToRead; i++) buffer[i] = globalInputText.c_str()[globalReadOffset + i];
     *numBytesRead = numBytesToRead;
     globalReadOffset += numBytesToRead;
 …
   IFilterExprNode* parseExpr(const string& strExpr)
+  {
+    #pragma omp critical (_parser)
+    {
+      if(globalInputText_ptr == 0) globalInputText_ptr = new std::string;
+      (*globalInputText_ptr).assign (strExpr);
+      globalReadOffset = 0;
+      yyparse();
+    }
+    globalInputText = strExpr;
+    globalReadOffset = 0;
+    yyparse();
     return parsed;
+  }
 …

XIOS/dev/branch_openmp/src/parse_expr/yacc_parser.yacc

-                      r1134
+                      r1328
+}
+  static IFilterExprNode* parsed;
+  static std::string globalInputText;
+  static std::string *globalInputText_ptr = 0;
+  static size_t globalReadOffset = 0;
+  #pragma omp threadprivate(parsed, globalInputText_ptr, globalReadOffset)
+  IFilterExprNode* parsed;
+  std::string globalInputText;
+  size_t globalReadOffset = 0;
   int readInputForLexer(char* buffer, size_t* numBytesRead, size_t maxBytesToRead)
+  {
-    if(globalInputText_ptr == 0) globalInputText_ptr = new std::string;
     size_t numBytesToRead = maxBytesToRead;
     size_t bytesRemaining = (*globalInputText_ptr).length()-globalReadOffset;
+    size_t bytesRemaining = globalInputText.length()-globalReadOffset;
     size_t i;
     if (numBytesToRead > bytesRemaining) numBytesToRead = bytesRemaining;
     for (i = 0; i < numBytesToRead; i++) buffer[i] = (*globalInputText_ptr).c_str()[globalReadOffset + i];
+    for (i = 0; i < numBytesToRead; i++) buffer[i] = globalInputText.c_str()[globalReadOffset + i];
     *numBytesRead = numBytesToRead;
     globalReadOffset += numBytesToRead;
 …
   IFilterExprNode* parseExpr(const string& strExpr)
+  {
+    #pragma omp critical (_parser)
+    {
+      if(globalInputText_ptr == 0) globalInputText_ptr = new std::string;
+      (*globalInputText_ptr).assign (strExpr);
+      globalReadOffset = 0;
+      yyparse();
+    }
+    globalInputText = strExpr;
+    globalReadOffset = 0;
+    yyparse();
     return parsed;
+  }

XIOS/dev/branch_openmp/src/policy.cpp

r855	r1328
10	10	#include "policy.hpp"
11	11	#include <cmath>
	12	using namespace ep_lib;
12	13
13	14	namespace xios

XIOS/dev/branch_openmp/src/registry.cpp

-                      r1134
+                      r1328
 #include "registry.hpp"
 #include "type.hpp"
+#include <mpi.hpp>
 #include <fstream>
 #include <sstream>
+using namespace ep_lib;
 namespace xios
 …
   void CRegistry::hierarchicalGatherRegistry(void)
+  {
+   // hierarchicalGatherRegistry(communicator) ;
+    gatherRegistry(communicator) ;
+    hierarchicalGatherRegistry(communicator) ;
+  }

XIOS/dev/branch_openmp/src/registry.hpp

-                      r1134
+                      r1328
 #include "mpi.hpp"
 #include "message.hpp"
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 // Those two headers can be replaced by the C++11 equivalent in the future
 …
 /** use internally for recursivity */
       void gatherRegistry(const MPI_Comm& comm) ;
+      void gatherRegistry(const ep_lib::MPI_Comm& comm) ;
 /** use internally for recursivity */
       void hierarchicalGatherRegistry(const MPI_Comm& comm) ;
+      void hierarchicalGatherRegistry(const ep_lib::MPI_Comm& comm) ;

XIOS/dev/branch_openmp/src/server.cpp

-                      r1205
+                      r1328
 #include <boost/functional/hash.hpp>
 #include <boost/algorithm/string.hpp>
+#include "mpi.hpp"
 #include "tracer.hpp"
 #include "timer.hpp"
 #include "event_scheduler.hpp"
+using namespace ep_lib;
 namespace xios
 …
     bool CServer::finished=false ;
     bool CServer::is_MPI_Initialized ;
     CEventScheduler* CServer::eventScheduler = 0;
     void CServer::initialize(void)
+    {
+     // int initialized ;
+     // MPI_Initialized(&initialized) ;
+     // if (initialized) is_MPI_Initialized=true ;
+     // else is_MPI_Initialized=false ;
       // Not using OASIS
       if (!CXios::usingOasis)
+      {
+       // if (!is_MPI_Initialized)
+       // {
+       //   MPI_Init(NULL, NULL);
+       // }
         CTimer::get("XIOS").resume() ;
 …
         unsigned long* hashAll ;
+//        int rank ;
         int size ;
         int myColor ;
 …
         MPI_Comm_size(CXios::globalComm,&size) ;
-        //size = CXios::globalComm.ep_comm_ptr->size_rank_info[0].second;
-        printf("global size = %d, size= %d\n", CXios::globalComm.ep_comm_ptr->size_rank_info[0].second, size);
         MPI_Comm_rank(CXios::globalComm,&rank);
         hashAll=new unsigned long[size] ;
 …
         myColor=colors[hashServer] ;
         MPI_Comm_split(CXios::globalComm,myColor,rank,&intraComm) ;
         int serverLeader=leaders[hashServer] ;
         int clientLeader;
          serverLeader=leaders[hashServer] ;
          for(it=leaders.begin();it!=leaders.end();++it)
+         for(it=leaders.begin();it!=leaders.end();it++)
+         {
            if (it->first!=hashServer)
 …
                      <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
-            // test_sendrecv(CXios::globalComm);
              MPI_Intercomm_create(intraComm,0,CXios::globalComm,clientLeader,0,&newComm) ;
              interComm.push_back(newComm) ;
 …
+      }
       // using OASIS
       else
+/*      else
+      {
+        int rank ,size;
         int size;
         if (!is_MPI_Initialized) oasis_init(CXios::xiosCodeId);
 …
               oasis_enddef() ;
+      }
+*/
+//      int rank;
       MPI_Comm_rank(intraComm,&rank) ;
       if (rank==0) isRoot=true;
 …
       delete eventScheduler ;
       for (std::list<MPI_Comm>::iterator it = contextInterComms.begin(); it != contextInterComms.end(); ++it)
+      for (std::list<MPI_Comm>::iterator it = contextInterComms.begin(); it != contextInterComms.end(); it++)
         MPI_Comm_free(&(*it));
       for (std::list<MPI_Comm>::iterator it = interComm.begin(); it != interComm.end(); ++it)
+      for (std::list<MPI_Comm>::iterator it = interComm.begin(); it != interComm.end(); it++)
         MPI_Comm_free(&(*it));
       MPI_Comm_free(&intraComm);
 …
+      {
         if (CXios::usingOasis) oasis_finalize();
         //else  {MPI_Finalize() ;}
+        //else MPI_Finalize() ;
+      }
       report(0)<<"Performance report : Time spent for XIOS : "<<CTimer::get("XIOS server").getCumulatedTime()<<endl  ;
       report(0)<<"Performance report : Time spent in processing events : "<<CTimer::get("Process events").getCumulatedTime()<<endl  ;
 …
+         {
            listenRootContext();
+           if (!finished)
+           {
+             listenRootFinalize() ;
+           }
+           if (!finished) listenRootFinalize() ;
+         }
          contextEventLoop() ;
          if (finished && contextList.empty()) stop=true ;
          eventScheduler->checkEvent() ;
+       }
        CTimer::get("XIOS server").suspend() ;
+     }
 …
         int flag ;
         for(it=interComm.begin();it!=interComm.end();++it)
+        for(it=interComm.begin();it!=interComm.end();it++)
+        {
            MPI_Status status ;
 …
               MPI_Recv(&msg,1,MPI_INT,0,0,*it,&status) ;
               info(20)<<" CServer : Receive client finalize"<<endl ;
               MPI_Comm_free(&(*it));
               interComm.erase(it) ;
 …
+       {
          traceOff() ;
+         #ifdef _usingEP
+         MPI_Iprobe(-1,1,CXios::globalComm, &flag, &status) ;
+         #else
+         MPI_Iprobe(MPI_ANY_SOURCE,1,CXios::globalComm, &flag, &status) ;
+         #endif
+         MPI_Iprobe(-2,1,CXios::globalComm, &flag, &status) ;
          traceOn() ;
          if (flag==true)
+         {
 …
            rank=status.MPI_SOURCE ;
            #elif _usingEP
            rank= status.ep_src ;
+           rank=status.ep_src;
            #endif
            MPI_Get_count(&status,MPI_CHAR,&count) ;
 …
            rank=status.MPI_SOURCE ;
            #elif _usingEP
            rank= status.ep_src ;
+           rank=status.ep_src;
            #endif
            MPI_Get_count(&status,MPI_CHAR,&count) ;
 …
             MPI_Isend(buff,count,MPI_CHAR,i,2,intraComm,&requests[i-1]) ;
+         }
          MPI_Waitall(size-1,requests,status) ;
          registerContext(buff,count,it->second.leaderRank) ;
 …
        bool finished ;
        map<string,CContext*>::iterator it ;
        for(it=contextList.begin();it!=contextList.end();++it)
+       for(it=contextList.begin();it!=contextList.end();it++)
+       {
          finished=it->second->checkBuffersAndListen();

XIOS/dev/branch_openmp/src/server.hpp

-                      r1134
+                      r1328
 #include "mpi.hpp"
 #include "event_scheduler.hpp"
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 namespace xios
 …
         static void registerContext(void* buff,int count, int leaderRank=0);
         static MPI_Comm intraComm;
         static list<MPI_Comm> interComm;
         static std::list<MPI_Comm> contextInterComms;
+        static ep_lib::MPI_Comm intraComm;
+        static list<ep_lib::MPI_Comm> interComm;
+        static std::list<ep_lib::MPI_Comm> contextInterComms;
         static CEventScheduler* eventScheduler;

XIOS/dev/branch_openmp/src/test/test_client.f90

-                      r1134
+                      r1328
   CALL MPI_INIT(ierr)
   CALL init_wait
-  CALL MPI_COMM_RANK(MPI_COMM_WORLD,rank,ierr)
-  if(rank < 2) then
 !!! XIOS Initialization (get the local communicator)
 …
   CALL xios_context_initialize("test",comm)
   CALL xios_get_handle("test",ctx_hdl)
   CALL xios_set_current_context(ctx_hdl)
 …
   CALL xios_is_defined_field_attr("field_A",enabled=ok)
   PRINT *,"field_A : attribute enabled is defined ? ",ok
   CALL xios_close_context_definition()
   PRINT*,"field field_A is active ? ",xios_field_is_active("field_A")
-  call MPI_Barrier(comm, ierr)
   DO ts=1,24*10
     CALL xios_update_calendar(ts)
     CALL xios_send_field("field_A",field_A)
     CALL wait_us(5000)
+    CALL wait_us(5000) ;
   ENDDO
 …
   CALL xios_finalize()
-  print *, "Client : xios_finalize "
-    else
-    CALL xios_init_server
-    print *, "Server : xios_finalize "
-    endif
   CALL MPI_FINALIZE(ierr)

XIOS/dev/branch_openmp/src/test/test_complete.f90

-                      r1134
+                      r1328
   IMPLICIT NONE
   INCLUDE "mpif.h"
   INTEGER :: rank, size
+  INTEGER :: rank
   INTEGER :: size_loc
   INTEGER :: ierr
 …
   INTEGER, ALLOCATABLE :: kindex(:)
   INTEGER :: ni,ibegin,iend,nj,jbegin,jend
   INTEGER :: i,j,l,ts,n, nb_pt, provided
+  INTEGER :: i,j,l,ts,n, nb_pt
 !!! MPI Initialization
+  CALL MPI_INIT_THREAD(3, provided, ierr)
+    if(provided .NE. 3) then
+      print*, "provided thread level = ", provided
+      call MPI_Abort()
+    endif
+  CALL MPI_INIT(ierr)
   CALL init_wait
-  CALL MPI_COMM_RANK(MPI_COMM_WORLD,rank,ierr)
-  CALL MPI_COMM_SIZE(MPI_COMM_WORLD,size,ierr)
-  if(rank < size-1) then
 !!! XIOS Initialization (get the local communicator)
 …
 !####################################################################################
+    DO ts=1,24*2
+    !DO ts=1,24
+    DO ts=1,24*10
       CALL xios_get_handle("atmosphere",ctx_hdl)
 …
 !!! Fin des contextes
+    CALL xios_get_handle("surface",ctx_hdl)
+    CALL xios_context_finalize()
+    CALL xios_get_handle("atmosphere",ctx_hdl)
     CALL xios_set_current_context(ctx_hdl)
     CALL xios_context_finalize()
-    print *, "xios_context_finalize(surface)"
-     CALL xios_get_handle("atmosphere",ctx_hdl)
-     CALL xios_set_current_context(ctx_hdl)
-     CALL xios_context_finalize()
-     print *, "xios_context_finalize(atmosphere)"
-!!! Fin de XIOS
-    CALL xios_finalize()
     DEALLOCATE(lon, lat, field_A_atm, lonvalue)
     DEALLOCATE(kindex, field_A_srf)
+     print *, "Client : xios_finalize "
+!!! Fin de XIOS
     CALL MPI_COMM_FREE(comm, ierr)
+  else
+    CALL xios_init_server
+    print *, "Server : xios_finalize "
+    endif
+    CALL xios_finalize()
     CALL MPI_FINALIZE(ierr)

XIOS/dev/branch_openmp/src/test/test_remap.f90

-                      r1141
+                      r1328
   CALL MPI_INIT(ierr)
   CALL init_wait
-  CALL MPI_COMM_RANK(MPI_COMM_WORLD,rank,ierr)
-  CALL MPI_COMM_SIZE(MPI_COMM_WORLD,size,ierr)
-  if(rank < size-2) then
 !!! XIOS Initialization (get the local communicator)
 …
   CALL xios_finalize()
-  print *, "Client : xios_finalize "
-    else
-    CALL xios_init_server
-    print *, "Server : xios_finalize "
-    endif
   CALL MPI_FINALIZE(ierr)

XIOS/dev/branch_openmp/src/timer.cpp

-                      r1205
+                      r1328
 #include <sstream>
 #include "tracer.hpp"
+using namespace ep_lib;
 namespace xios
+{
   std::map<std::string,CTimer> CTimer::allTimer;
+  //std::map<std::string,CTimer> CTimer::allTimer;
   std::map<std::string,CTimer> *CTimer::allTimer_ptr = 0;
 …
   CTimer& CTimer::get(const std::string name)
+  {
+    // bkp
+    // std::map<std::string,CTimer>::iterator it = allTimer.find(name);
+    // if (it == allTimer.end())
+    //   it = allTimer.insert(std::make_pair(name, CTimer(name))).first;
+    // return it->second;
+    if(allTimer_ptr == NULL) allTimer_ptr = new std::map<std::string,CTimer>;
+    if(allTimer_ptr == 0) allTimer_ptr = new std::map<std::string,CTimer>;
+    //std::map<std::string,CTimer>::iterator it = allTimer.find(name);
+    std::map<std::string,CTimer>::iterator it = allTimer_ptr->find(name);
+    //if (it == allTimer.end())
+    if (it == allTimer_ptr->end())
+      it = allTimer_ptr->insert(std::make_pair(name, CTimer(name))).first;
+    std::map<std::string,CTimer>::iterator it = (*allTimer_ptr).find(name);
+    if (it == (*allTimer_ptr).end())
+      it = (*allTimer_ptr).insert(std::make_pair(name, CTimer(name))).first;
+      //it = allTimer.insert(std::make_pair(name, CTimer(name))).first;
     return it->second;
+  }
 …
+  {
     std::ostringstream strOut ;
+    for(std::map<std::string,CTimer>::iterator it=allTimer.begin();it!=allTimer.end();++it)
+    if(allTimer_ptr == 0) allTimer_ptr = new std::map<std::string,CTimer>;
+    //for(std::map<std::string,CTimer>::iterator it=allTimer.begin();it!=allTimer.end();++it)
+    for(std::map<std::string,CTimer>::iterator it=allTimer_ptr->begin();it!=allTimer_ptr->end();++it)
       strOut<<"Timer : "<<it->first<<"    -->   cumulated time : "<<it->second.getCumulatedTime()<<std::endl ;
     return strOut.str() ;

XIOS/dev/branch_openmp/src/timer.hpp

-                      r1205
+                      r1328
       void reset(void);
       double getCumulatedTime(void);
+      static std::map<std::string,CTimer> allTimer;
+      //static std::map<std::string,CTimer> allTimer;
       static std::map<std::string,CTimer> *allTimer_ptr;
-      #pragma omp threadprivate(allTimer_ptr)
       static double getTime(void);
       static CTimer& get(std::string name);

XIOS/dev/branch_openmp/src/transformation/Functions/average_reduction.cpp

-                      r1205
+                      r1328
                                        CArray<double,1>& dataOut,
                                        std::vector<bool>& flagInitial,
                                        bool ignoreMissingValue)
+                                       bool ignoreMissingValue, bool firstPass)
+{
   if (resetWeight_) { weights_.resize(flagInitial.size()); weights_ = 1.0; resetWeight_ = false; }
 …
     int nbLocalIndex = localIndex.size();
     int currentlocalIndex = 0;
+    double currentWeight  = 0.0;
+    double currentWeight  = 0.0;
+    if (firstPass) dataOut=std::numeric_limits<double>::quiet_NaN();
     for (int idx = 0; idx < nbLocalIndex; ++idx)
 …
           weights_(currentlocalIndex) += 1.0;
+        }
+      }
-      else
+      {
-        if (flagInitial[currentlocalIndex])
-          dataOut(currentlocalIndex) = std::numeric_limits<double>::quiet_NaN();
+      }
+    }

XIOS/dev/branch_openmp/src/transformation/Functions/average_reduction.hpp

r1076	r1328
27	27	CArray<double,1>& dataOut,
28	28	std::vector<bool>& flagInitial,
29		bool ignoreMissingValue);
	29	bool ignoreMissingValue, bool firstPass);
30	30
31	31	virtual void updateData(CArray<double,1>& dataOut);

XIOS/dev/branch_openmp/src/transformation/Functions/extract.cpp

r1076	r1328
30	30	CArray<double,1>& dataOut,
31	31	std::vector<bool>& flagInitial,
32		bool ignoreMissingValue)
	32	bool ignoreMissingValue, bool firstPass)
33	33	{
34	34	int nbLocalIndex = localIndex.size();

XIOS/dev/branch_openmp/src/transformation/Functions/extract.hpp

r1076	r1328
27	27	CArray<double,1>& dataOut,
28	28	std::vector<bool>& flagInitial,
29		bool ignoreMissingValue);
	29	bool ignoreMissingValue, bool firstPass);
30	30
31	31	virtual ~CExtractReductionAlgorithm() {}

XIOS/dev/branch_openmp/src/transformation/Functions/max_reduction.cpp

r1205	r1328
31	31	CArray<double,1>& dataOut,
32	32	std::vector<bool>& flagInitial,
33		bool ignoreMissingValue)
	33	bool ignoreMissingValue, bool firstPass)
34	34	{
35	35	if (ignoreMissingValue)
36	36	{
37	37	int nbLocalIndex = localIndex.size();
38		int currentlocalIndex = 0;
	38	int currentlocalIndex = 0;
	39	if (firstPass) dataOut=std::numeric_limits<double>::quiet_NaN();
39	40	for (int idx = 0; idx < nbLocalIndex; ++idx)
40	41	{
…	…
51	52	dataOut(currentlocalIndex) = std::max(*(dataInput + idx), dataOut(currentlocalIndex));
52	53	}
53		}
54		~~else~~
55		{
56		~~if (flagInitial[currentlocalIndex])~~
57		~~dataOut(currentlocalIndex) = std::numeric_limits<double>::quiet_NaN();~~
58	54	}
59	55	}

XIOS/dev/branch_openmp/src/transformation/Functions/max_reduction.hpp

r1076	r1328
27	27	CArray<double,1>& dataOut,
28	28	std::vector<bool>& flagInitial,
29		bool ignoreMissingValue);
	29	bool ignoreMissingValue, bool firstPass);
30	30
31	31	virtual ~CMaxReductionAlgorithm() {}

XIOS/dev/branch_openmp/src/transformation/Functions/min_reduction.cpp

r1205	r1328
31	31	CArray<double,1>& dataOut,
32	32	std::vector<bool>& flagInitial,
33		bool ignoreMissingValue)
	33	bool ignoreMissingValue, bool firstPass)
34	34	{
35	35	if (ignoreMissingValue)
36	36	{
37	37	int nbLocalIndex = localIndex.size();
38		int currentlocalIndex = 0;
	38	int currentlocalIndex = 0;
	39	if (firstPass) dataOut=std::numeric_limits<double>::quiet_NaN();
39	40	for (int idx = 0; idx < nbLocalIndex; ++idx)
40	41	{
…	…
51	52	dataOut(currentlocalIndex) = std::min(*(dataInput + idx), dataOut(currentlocalIndex));
52	53	}
53		}
54		~~else~~
55		{
56		~~if (flagInitial[currentlocalIndex])~~
57		~~dataOut(currentlocalIndex) = std::numeric_limits<double>::quiet_NaN();~~
58	54	}
59	55	}

XIOS/dev/branch_openmp/src/transformation/Functions/min_reduction.hpp

r1076	r1328
27	27	CArray<double,1>& dataOut,
28	28	std::vector<bool>& flagInitial,
29		bool ignoreMissingValue);
	29	bool ignoreMissingValue, bool firstPass);
30	30
31	31	virtual ~CMinReductionAlgorithm() {}

XIOS/dev/branch_openmp/src/transformation/Functions/reduction.cpp

-                      r1155
+                      r1328
 CReductionAlgorithm::CallBackMap* CReductionAlgorithm::reductionCreationCallBacks_ = 0;
 //std::map<StdString,EReductionType> CReductionAlgorithm::ReductionOperations = std::map<StdString,EReductionType>();
+std::map<StdString,EReductionType> *CReductionAlgorithm::ReductionOperations_ptr = 0;
+//#pragma omp threadprivate(CReductionAlgorithm::ReductionOperations_ptr)
+std::map<StdString,EReductionType> *CReductionAlgorithm::ReductionOperations_ptr = 0;
+bool CReductionAlgorithm::initReductionOperation(std::map<StdString,EReductionType>& m)
+{
+  // So so stupid way to intialize operation but it works ...
+  m["sum"] = TRANS_REDUCE_SUM;
+  CSumReductionAlgorithm::registerTrans();
+  m["min"] = TRANS_REDUCE_MIN;
+  CMinReductionAlgorithm::registerTrans();
+  m["max"] = TRANS_REDUCE_MAX;
+  CMaxReductionAlgorithm::registerTrans();
+  m["extract"] = TRANS_REDUCE_EXTRACT;
+  CExtractReductionAlgorithm::registerTrans();
+  m["average"] = TRANS_REDUCE_AVERAGE;
+  CAverageReductionAlgorithm::registerTrans();
+}
 bool CReductionAlgorithm::initReductionOperation()
 …
 //bool CReductionAlgorithm::_dummyInit = CReductionAlgorithm::initReductionOperation(CReductionAlgorithm::ReductionOperations);
 //bool CReductionAlgorithm::_dummyInit = CReductionAlgorithm::initReductionOperation();
+bool CReductionAlgorithm::_dummyInit = CReductionAlgorithm::initReductionOperation();
 CReductionAlgorithm* CReductionAlgorithm::createOperation(EReductionType reduceType)
+{
   int reduceTypeInt = reduceType;
   CallBackMap::const_iterator it = (*reductionCreationCallBacks_).find(reduceType);
   if ((*reductionCreationCallBacks_).end() == it)

XIOS/dev/branch_openmp/src/transformation/Functions/reduction.hpp

-                      r1155
+                      r1328
   //static std::map<StdString,EReductionType> ReductionOperations;
   static std::map<StdString,EReductionType> *ReductionOperations_ptr;
-  #pragma omp threadprivate(ReductionOperations_ptr)
 public:
   CReductionAlgorithm() { }
+  CReductionAlgorithm() {}
   /*!
 …
     \param [in/out] dataOut Array contains local data
     \param [in/out] flagInitial vector of boolean to mark the local index already initialized. True means there is a need for initialization
+    \param [in] firstPass indicate if it is the first time the apply funtion is called for a same transformation, in order to make a clean initialization
   */
   virtual void apply(const std::vector<std::pair<int,double> >& localIndex,
 …
                      CArray<double,1>& dataOut,
                      std::vector<bool>& flagInitial,
                      bool ignoreMissingValue) = 0;
+                     bool ignoreMissingValue, bool firstPass) = 0;
   /*!
     Update local data
 …
   typedef std::map<EReductionType, CreateOperationCallBack> CallBackMap;
   static CallBackMap* reductionCreationCallBacks_;
-  #pragma omp threadprivate(reductionCreationCallBacks_)
   static bool registerOperation(EReductionType reduceType, CreateOperationCallBack createFn);
 …
   static bool initReductionOperation();
   static bool _dummyInit;
-  #pragma omp threadprivate(_dummyInit)
 };

XIOS/dev/branch_openmp/src/transformation/Functions/sum_reduction.cpp

r1205	r1328
31	31	CArray<double,1>& dataOut,
32	32	std::vector<bool>& flagInitial,
33		bool ignoreMissingValue)
	33	bool ignoreMissingValue, bool firstPass)
34	34	{
35	35	if (ignoreMissingValue)
36	36	{
37	37	int nbLocalIndex = localIndex.size();
38		int currentlocalIndex = 0;
	38	int currentlocalIndex = 0;
	39
	40	if (firstPass) dataOut=std::numeric_limits<double>::quiet_NaN();
39	41
40	42	for (int idx = 0; idx < nbLocalIndex; ++idx)
…	…
52	54	dataOut(currentlocalIndex) += *(dataInput + idx);
53	55	}
54		}
55		~~else~~
56		{
57		~~if (flagInitial[currentlocalIndex])~~
58		~~dataOut(currentlocalIndex) = std::numeric_limits<double>::quiet_NaN();~~
59	56	}
60	57	}

XIOS/dev/branch_openmp/src/transformation/Functions/sum_reduction.hpp

r1076	r1328
27	27	CArray<double,1>& dataOut,
28	28	std::vector<bool>& flagInitial,
29		bool ignoreMissingValue);
	29	bool ignoreMissingValue, bool firstPass);
30	30
31	31	virtual ~CSumReductionAlgorithm() {}

XIOS/dev/branch_openmp/src/transformation/axis_algorithm_extract_domain.cpp

-                      r1155
+                      r1328
 #include "grid.hpp"
 #include "grid_transformation_factory_impl.hpp"
+#include "reduction.hpp"
 namespace xios {
 …
   pos_ = algo->position;
+  if(CReductionAlgorithm::ReductionOperations_ptr == 0)
+    CReductionAlgorithm::initReductionOperation();
+  reduction_ = CReductionAlgorithm::createOperation((*CReductionAlgorithm::ReductionOperations_ptr)[op]);
+  //reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations[op]);
+  reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations_ptr->at(op));
+}
 …
                                         CArray<double,1>& dataOut,
                                         std::vector<bool>& flagInitial,
                                         bool ignoreMissingValue)
+                                        bool ignoreMissingValue, bool firstPass)
+{
   reduction_->apply(localIndex, dataInput, dataOut, flagInitial, ignoreMissingValue);
+  reduction_->apply(localIndex, dataInput, dataOut, flagInitial, ignoreMissingValue, firstPass);
+}

XIOS/dev/branch_openmp/src/transformation/axis_algorithm_extract_domain.hpp

-                      r1155
+                      r1328
 #include "axis_algorithm_transformation.hpp"
 #include "transformation.hpp"
-#include "reduction.hpp"
 namespace xios {
 …
   Extract a domain to an axis
 */
 class CAxisAlgorithmExtractDomain : public CAxisAlgorithmTransformation, public CReductionAlgorithm
+class CAxisAlgorithmExtractDomain : public CAxisAlgorithmTransformation
+{
 public:
 …
                      CArray<double,1>& dataOut,
                      std::vector<bool>& flagInitial,
                      bool ignoreMissingValue);
+                     bool ignoreMissingValue, bool firstPass);
   virtual ~CAxisAlgorithmExtractDomain();

XIOS/dev/branch_openmp/src/transformation/axis_algorithm_inverse.cpp

-                      r1134
+                      r1328
 #include "inverse_axis.hpp"
 #include "client_client_dht_template.hpp"
+using namespace ep_lib;
 namespace xios {
 …
   // Sending global index of grid source to corresponding process as well as the corresponding mask
   std::vector<ep_lib::MPI_Request> requests;
   std::vector<ep_lib::MPI_Status> status;
+  std::vector<MPI_Request> requests;
+  std::vector<MPI_Status> status;
   boost::unordered_map<int, unsigned long* > recvGlobalIndexSrc;
   boost::unordered_map<int, double* > sendValueToDest;
 …
     sendValueToDest[recvRank] = new double [recvSize];
     requests.push_back(ep_lib::MPI_Request());
+    requests.push_back(MPI_Request());
     MPI_Irecv(recvGlobalIndexSrc[recvRank], recvSize, MPI_UNSIGNED_LONG, recvRank, 46, client->intraComm, &requests.back());
+  }
 …
     // Send global index source and mask
     requests.push_back(ep_lib::MPI_Request());
+    requests.push_back(MPI_Request());
     MPI_Isend(sendGlobalIndexSrc[sendRank], sendSize, MPI_UNSIGNED_LONG, sendRank, 46, client->intraComm, &requests.back());
+  }
 …
   MPI_Waitall(requests.size(), &requests[0], &status[0]);
+  std::vector<ep_lib::MPI_Request>().swap(requests);
+  std::vector<ep_lib::MPI_Status>().swap(status);
+  std::vector<MPI_Request>().swap(requests);
+  std::vector<MPI_Status>().swap(status);
   // Okie, on destination side, we will wait for information of masked index of source
 …
     int recvSize = itSend->second;
     requests.push_back(ep_lib::MPI_Request());
+    requests.push_back(MPI_Request());
     MPI_Irecv(recvValueFromSrc[recvRank], recvSize, MPI_DOUBLE, recvRank, 48, client->intraComm, &requests.back());
+  }
 …
+    }
     // Okie, now inform the destination which source index are masked
     requests.push_back(ep_lib::MPI_Request());
+    requests.push_back(MPI_Request());
     MPI_Isend(sendValueToDest[recvRank], recvSize, MPI_DOUBLE, recvRank, 48, client->intraComm, &requests.back());
+  }
   status.resize(requests.size());
   MPI_Waitall(requests.size(), &requests[0], &status[0]);
   size_t nGloAxisDest = axisDest_->n_glo.getValue() - 1;

XIOS/dev/branch_openmp/src/transformation/axis_algorithm_inverse.hpp

-                      r1134
+                      r1328
 #include "axis_algorithm_transformation.hpp"
 #include "transformation.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 namespace xios {

XIOS/dev/branch_openmp/src/transformation/axis_algorithm_reduce_domain.cpp

-                      r1155
+                      r1328
 #include "grid.hpp"
 #include "grid_transformation_factory_impl.hpp"
+#include "reduction.hpp"
 namespace xios {
 …
   dir_ = (CReduceDomainToAxis::direction_attr::iDir == algo->direction)  ? iDir : jDir;
+  if(CReductionAlgorithm::ReductionOperations_ptr == 0)
+    CReductionAlgorithm::initReductionOperation();
+  reduction_ = CReductionAlgorithm::createOperation((*CReductionAlgorithm::ReductionOperations_ptr)[op]);
+  //reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations[op]);
+  reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations_ptr->at(op));
+}
 …
                                        CArray<double,1>& dataOut,
                                        std::vector<bool>& flagInitial,
                                        bool ignoreMissingValue)
+                                       bool ignoreMissingValue, bool firstPass)
+{
   reduction_->apply(localIndex, dataInput, dataOut, flagInitial, ignoreMissingValue);
+  reduction_->apply(localIndex, dataInput, dataOut, flagInitial, ignoreMissingValue, firstPass);
+}
 …
   CArray<int,1>& axisDstIndex = axisDest_->index;
   int ni_glo = domainSrc_->ni_glo, nj_glo = domainSrc_->nj_glo;
   if (jDir == dir_)
+  if (iDir == dir_)
+  {
     int nbAxisIdx = axisDstIndex.numElements();
 …
+    }
+  }
   else if (iDir == dir_)
+  else if (jDir == dir_)
+  {
     int nbAxisIdx = axisDstIndex.numElements();

XIOS/dev/branch_openmp/src/transformation/axis_algorithm_reduce_domain.hpp

-                      r1155
+                      r1328
 #include "axis_algorithm_transformation.hpp"
 #include "transformation.hpp"
-#include "reduction.hpp"
 namespace xios {
 …
   Reduce a domain to an axis
 */
 class CAxisAlgorithmReduceDomain : public CAxisAlgorithmTransformation, public CReductionAlgorithm
+class CAxisAlgorithmReduceDomain : public CAxisAlgorithmTransformation
+{
 public:
 …
                      CArray<double,1>& dataOut,
                      std::vector<bool>& flagInitial,
                      bool ignoreMissingValue);
+                     bool ignoreMissingValue, bool firstPass);
   virtual void updateData(CArray<double,1>& dataOut);

XIOS/dev/branch_openmp/src/transformation/axis_algorithm_zoom.cpp

r1205	r1328
44	44	zoomBegin_ = zoomAxis->begin.getValue();
45	45	zoomSize_ = zoomAxis->n.getValue();
46		zoomEnd_ = zoomBegin_ + zoomSize_ - 1;
	46	zoomEnd_ = zoomBegin_ + zoomSize_ - 1;
47	47
48	48	if (zoomSize_ > axisSource->n_glo.getValue())

XIOS/dev/branch_openmp/src/transformation/domain_algorithm_interpolate.cpp

-                      r1205
+                      r1328
 #include "interpolate_domain.hpp"
 #include "grid.hpp"
+using namespace ep_lib;
 namespace xios {
 …
   nVertexSrc = nVertexDest = constNVertex;
   // First of all, try to retrieve the boundary values of domain source and domain destination
   int localDomainSrcSize = domainSrc_->i_index.numElements();
 …
+  }
   for (int idx = 0; idx < nDstLocal; ++idx)
+  {
 …
   CContext* context = CContext::getCurrent();
   CContextClient* client=context->client;
+  int split_key;
+  ep_lib::MPI_Comm_rank(client->intraComm, &split_key);
+  ep_lib::MPI_Comm poleComme(MPI_COMM_NULL);
+  ep_lib::MPI_Comm_split(client->intraComm, interMapValuePole.empty() ? 9 : 1, split_key, &poleComme);
+  if (MPI_COMM_NULL != poleComme)
+  ep_lib::MPI_Comm poleComme;
+  ep_lib::MPI_Comm_split(client->intraComm, interMapValuePole.empty() ? 0 : 1, 0, &poleComme);
+  if (!poleComme.is_null())
+  {
     int nbClientPole;
 …
     for(int n=1;n<nbClientPole;++n) displ[n]=displ[n-1]+recvCount[n-1] ;
     int recvSize=displ[nbClientPole-1]+recvCount[nbClientPole-1] ;
     std::vector<int> sendSourceIndexBuff(nbWeight);
 …
   double* sendWeightBuff = new double [sendBuffSize];
   std::vector<ep_lib::MPI_Request> sendRequest;
+  std::vector<ep_lib::MPI_Request> sendRequest(3*globalIndexInterpSendToClient.size());
   int sendOffSet = 0, l = 0;
+  int position = 0;
   for (itMap = itbMap; itMap != iteMap; ++itMap)
+  {
 …
+    }
-    sendRequest.push_back(ep_lib::MPI_Request());
     ep_lib::MPI_Isend(sendIndexDestBuff + sendOffSet,
              k,
 …
              MPI_DOMAIN_INTERPOLATION_DEST_INDEX,
              client->intraComm,
+             &sendRequest.back());
+    sendRequest.push_back(ep_lib::MPI_Request());
+             &sendRequest[position++]);
     ep_lib::MPI_Isend(sendIndexSrcBuff + sendOffSet,
              k,
 …
              MPI_DOMAIN_INTERPOLATION_SRC_INDEX,
              client->intraComm,
+             &sendRequest.back());
+    sendRequest.push_back(ep_lib::MPI_Request());
+             &sendRequest[position++]);
     ep_lib::MPI_Isend(sendWeightBuff + sendOffSet,
              k,
 …
              MPI_DOMAIN_INTERPOLATION_WEIGHT,
              client->intraComm,
              &sendRequest.back());
+             &sendRequest[position++]);
     sendOffSet += k;
+  }
 …
              recvBuffSize,
              MPI_INT,
              MPI_ANY_SOURCE,
+             -2,
              MPI_DOMAIN_INTERPOLATION_DEST_INDEX,
              client->intraComm,
 …
     clientSrcRank = recvStatus.ep_src;
     #endif
     ep_lib::MPI_Recv((recvIndexSrcBuff + receivedSize),
              recvBuffSize,
 …
   std::vector<ep_lib::MPI_Status> requestStatus(sendRequest.size());
+  ep_lib::MPI_Status stat_ignore;
+  ep_lib::MPI_Waitall(sendRequest.size(), &sendRequest[0], &stat_ignore);
+  ep_lib::MPI_Waitall(sendRequest.size(), &sendRequest[0], &requestStatus[0]);
   delete [] sendIndexDestBuff;
 …
 /*! Redefined some functions of CONetCDF4 to make use of them */
 CDomainAlgorithmInterpolate::WriteNetCdf::WriteNetCdf(const StdString& filename, const MPI_Comm comm)
+CDomainAlgorithmInterpolate::WriteNetCdf::WriteNetCdf(const StdString& filename, const ep_lib::MPI_Comm comm)
   : CNc4DataOutput(NULL, filename, false, false, true, comm, false, true) {}
 int CDomainAlgorithmInterpolate::WriteNetCdf::addDimensionWrite(const StdString& name,
 …
   std::vector<StdSize> start(1, startIndex - localNbWeight);
   std::vector<StdSize> count(1, localNbWeight);
   WriteNetCdf netCdfWriter(filename, static_cast<MPI_Comm>(client->intraComm.mpi_comm));
+  WriteNetCdf netCdfWriter(filename, client->intraComm);
   // Define some dimensions

XIOS/dev/branch_openmp/src/transformation/domain_algorithm_interpolate.hpp

-                      r1205
+                      r1328
 #ifndef __XIOS_DOMAIN_ALGORITHM_INTERPOLATE_HPP__
 #define __XIOS_DOMAIN_ALGORITHM_INTERPOLATE_HPP__
+#include "mpi_std.hpp"
 #include "domain_algorithm_transformation.hpp"
 #include "transformation.hpp"
 #include "nc4_data_output.hpp"
-#ifdef _usingEP
-#include "ep_declaration.hpp"
-#endif
 namespace xios {
 …
+  {
   public:
     WriteNetCdf(const StdString& filename, const MPI_Comm comm);
+    WriteNetCdf(const StdString& filename, const ep_lib::MPI_Comm comm);
     int addDimensionWrite(const StdString& name, const StdSize size = UNLIMITED_DIM);
     int addVariableWrite(const StdString& name, nc_type type,

XIOS/dev/branch_openmp/src/transformation/generic_algorithm_transformation.cpp

r1205	r1328
30	30	CArray<double,1>& dataOut,
31	31	std::vector<bool>& flagInitial,
32		bool ignoreMissingValue)
	32	bool ignoreMissingValue, bool firstPass )
33	33	{
34	34	int nbLocalIndex = localIndex.size();

XIOS/dev/branch_openmp/src/transformation/generic_algorithm_transformation.hpp

-                      r1076
+                      r1328
     \param [in/out] flagInitial vector of boolean to mark the local index already initialized. True means there is a need for initalization
     \param [in] ignoreMissingValue don't count missing value in operation if this flag is true
+    \param [in] firstPass indicate if it is the first time the apply funtion is called for a same transformation, in order to make a clean initialization
   */
   virtual void apply(const std::vector<std::pair<int,double> >& localIndex,
 …
                      CArray<double,1>& dataOut,
                      std::vector<bool>& flagInitial,
                      bool ignoreMissingValue);
+                     bool ignoreMissingValue, bool firstPass);
   /*!

XIOS/dev/branch_openmp/src/transformation/grid_transformation.cpp

-                      r1203
+                      r1328
     sendRankSizeMap[itIndex->first] = sendSize;
+  }
   MPI_Allgather(&connectedClient,1,MPI_INT,recvCount,1,MPI_INT,client->intraComm);
+  ep_lib::MPI_Allgather(&connectedClient,1,MPI_INT,recvCount,1,MPI_INT,client->intraComm);
   displ[0]=0 ;
 …
   int* recvRankBuff=new int[recvSize];
   int* recvSizeBuff=new int[recvSize];
   MPI_Allgatherv(sendRankBuff,connectedClient,MPI_INT,recvRankBuff,recvCount,displ,MPI_INT,client->intraComm);
   MPI_Allgatherv(sendSizeBuff,connectedClient,MPI_INT,recvSizeBuff,recvCount,displ,MPI_INT,client->intraComm);
+  ep_lib::MPI_Allgatherv(sendRankBuff,connectedClient,MPI_INT,recvRankBuff,recvCount,displ,MPI_INT,client->intraComm);
+  ep_lib::MPI_Allgatherv(sendSizeBuff,connectedClient,MPI_INT,recvSizeBuff,recvCount,displ,MPI_INT,client->intraComm);
   for (int i = 0; i < nbClient; ++i)
+  {
 …
   // Sending global index of grid source to corresponding process as well as the corresponding mask
   std::vector<ep_lib::MPI_Request> requests(2*recvRankSizeMap.size()+2*globaIndexWeightFromSrcToDst.size());
+  std::vector<ep_lib::MPI_Request> requests(recvRankSizeMap.size()*2 + globaIndexWeightFromSrcToDst.size()*2);
   std::vector<ep_lib::MPI_Status> status;
   boost::unordered_map<int, unsigned char* > recvMaskDst;
   boost::unordered_map<int, unsigned long* > recvGlobalIndexSrc;
   int position = 0;
+  int requests_position = 0;
   for (std::map<int,int>::const_iterator itRecv = recvRankSizeMap.begin(); itRecv != recvRankSizeMap.end(); ++itRecv)
+  {
 …
     recvGlobalIndexSrc[recvRank] = new unsigned long [recvSize];
+    MPI_Irecv(recvGlobalIndexSrc[recvRank], recvSize, MPI_UNSIGNED_LONG, recvRank, 46, client->intraComm, &requests[position]);
+    position++;
+    MPI_Irecv(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 47, client->intraComm, &requests[position]);
+    position++;
+    ep_lib::MPI_Irecv(recvGlobalIndexSrc[recvRank], recvSize, MPI_UNSIGNED_LONG, recvRank, 46, client->intraComm, &requests[requests_position++]);
+    ep_lib::MPI_Irecv(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 47, client->intraComm, &requests[requests_position++]);
+  }
 …
     // Send global index source and mask
+    MPI_Isend(sendGlobalIndexSrc[sendRank], sendSize, MPI_UNSIGNED_LONG, sendRank, 46, client->intraComm, &requests[position]);
+    position++;
+    MPI_Isend(sendMaskDst[sendRank], sendSize, MPI_UNSIGNED_CHAR, sendRank, 47, client->intraComm, &requests[position]);
+    position++;
+    ep_lib::MPI_Isend(sendGlobalIndexSrc[sendRank], sendSize, MPI_UNSIGNED_LONG, sendRank, 46, client->intraComm, &requests[requests_position++]);
+    ep_lib::MPI_Isend(sendMaskDst[sendRank], sendSize, MPI_UNSIGNED_CHAR, sendRank, 47, client->intraComm, &requests[requests_position++]);
+  }
   status.resize(requests.size());
   MPI_Waitall(requests.size(), &requests[0], &status[0]);
+  ep_lib::MPI_Waitall(requests.size(), &requests[0], &status[0]);
   // Okie, now use the mask to identify which index source we need to send, then also signal the destination which masked index we will return
+  //std::vector<ep_lib::MPI_Request>().swap(requests);
+  //std::vector<ep_lib::MPI_Status>().swap(status);
+  requests.resize(sendRankSizeMap.size()+recvRankSizeMap.size());
+  position = 0;
+  requests.resize(sendRankSizeMap.size() + recvRankSizeMap.size());
+  requests_position = 0;
+  std::vector<ep_lib::MPI_Status>().swap(status);
   // Okie, on destination side, we will wait for information of masked index of source
   for (std::map<int,int>::const_iterator itSend = sendRankSizeMap.begin(); itSend != sendRankSizeMap.end(); ++itSend)
 …
     int recvSize = itSend->second;
+    MPI_Irecv(sendMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests[position]);
+    position++;
+    ep_lib::MPI_Irecv(sendMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests[requests_position++]);
+  }
 …
     // Okie, now inform the destination which source index are masked
+    MPI_Isend(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests[position]);
+    position++;
+    ep_lib::MPI_Isend(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests[requests_position++]);
+  }
   status.resize(requests.size());
   MPI_Waitall(requests.size(), &requests[0], &status[0]);
+  ep_lib::MPI_Waitall(requests.size(), &requests[0], &status[0]);
   // Cool, now we can fill in local index of grid destination (counted for masked index)

XIOS/dev/branch_openmp/src/transformation/grid_transformation.hpp

r978	r1328
12	12	#include <map>
13	13	#include <vector>
	14	#include "mpi_std.hpp"
14	15	#include "generic_algorithm_transformation.hpp"
15	16	#include "transformation_enum.hpp"

XIOS/dev/branch_openmp/src/transformation/grid_transformation_factory_impl.hpp

-                      r1134
+                      r1328
   typedef std::map<ETranformationType, CreateTransformationCallBack> CallBackMap;
   static CallBackMap* transformationCreationCallBacks_;
-  #pragma omp threadprivate(transformationCreationCallBacks_)
   static bool registerTransformation(ETranformationType transType, CreateTransformationCallBack createFn);
   static bool unregisterTransformation(ETranformationType transType);
   static bool initializeTransformation_;
-  #pragma omp threadprivate(initializeTransformation_)
 };
 …
                                                                                std::map<int, int>& elementPositionInGridDst2DomainPosition)
+{
-  if (0 == transformationCreationCallBacks_)
-    transformationCreationCallBacks_ = new CallBackMap();
   typename CallBackMap::const_iterator it = (*transformationCreationCallBacks_).find(transType);
   if ((*transformationCreationCallBacks_).end() == it)

XIOS/dev/branch_openmp/src/transformation/grid_transformation_selector.cpp

r1106	r1328
10	10	#include "grid.hpp"
11	11	#include "algo_types.hpp"
	12	using namespace ep_lib;
12	13
13	14	namespace xios {

XIOS/dev/branch_openmp/src/transformation/grid_transformation_selector.hpp

r978	r1328
12	12	#include <map>
13	13	#include <vector>
	14	#include "mpi_std.hpp"
14	15	#include "generic_algorithm_transformation.hpp"
15	16	#include "transformation_enum.hpp"

XIOS/dev/branch_openmp/src/transformation/scalar_algorithm_extract_axis.cpp

-                      r1155
+                      r1328
 #include "grid_transformation_factory_impl.hpp"
+#include "reduction.hpp"
 namespace xios {
 …
   StdString op = "extract";
   pos_ = algo->position;
+  if(CReductionAlgorithm::ReductionOperations_ptr == 0)
+    CReductionAlgorithm::initReductionOperation();
+  reduction_ = CReductionAlgorithm::createOperation((*CReductionAlgorithm::ReductionOperations_ptr)[op]);
+  //reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations[op]);
+  reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations_ptr->at(op));
+}
 …
                                          CArray<double,1>& dataOut,
                                          std::vector<bool>& flagInitial,
                                          bool ignoreMissingValue)
+                                         bool ignoreMissingValue, bool firstPass)
+{
   reduction_->apply(localIndex, dataInput, dataOut, flagInitial, ignoreMissingValue);
+  reduction_->apply(localIndex, dataInput, dataOut, flagInitial, ignoreMissingValue, firstPass);
+}

XIOS/dev/branch_openmp/src/transformation/scalar_algorithm_extract_axis.hpp

-                      r1155
+                      r1328
 #include "scalar_algorithm_transformation.hpp"
 #include "transformation.hpp"
-#include "reduction.hpp"
 namespace xios {
 …
   Extract a scalar from an axis
 */
 class CScalarAlgorithmExtractAxis : public CScalarAlgorithmTransformation, public CReductionAlgorithm
+class CScalarAlgorithmExtractAxis : public CScalarAlgorithmTransformation
+{
 public:
 …
                      CArray<double,1>& dataOut,
                      std::vector<bool>& flagInitial,
                      bool ignoreMissingValue);
+                     bool ignoreMissingValue, bool firstPass);
   virtual ~CScalarAlgorithmExtractAxis();

XIOS/dev/branch_openmp/src/transformation/scalar_algorithm_reduce_axis.cpp

-                      r1155
+                      r1328
 #include "grid.hpp"
 #include "grid_transformation_factory_impl.hpp"
+#include "reduction.hpp"
+#include "reduction.hpp"
 namespace xios {
-//extern std::map<StdString,EReductionType> *CReductionAlgorithm::ReductionOperations_ptr;
-//#pragma omp threadprivate(CReductionAlgorithm::ReductionOperations_ptr)
 CGenericAlgorithmTransformation* CScalarAlgorithmReduceAxis::create(CGrid* gridDst, CGrid* gridSrc,
                                                                      CTransformation<CScalar>* transformation,
 …
+  }
+  if(CReductionAlgorithm::ReductionOperations_ptr == 0)
+    CReductionAlgorithm::initReductionOperation();
+  if ((*CReductionAlgorithm::ReductionOperations_ptr).end() == (*CReductionAlgorithm::ReductionOperations_ptr).find(op))
+  //if (CReductionAlgorithm::ReductionOperations.end() == CReductionAlgorithm::ReductionOperations.find(op))
+  if (CReductionAlgorithm::ReductionOperations_ptr->end() == CReductionAlgorithm::ReductionOperations_ptr->find(op))
     ERROR("CScalarAlgorithmReduceAxis::CScalarAlgorithmReduceAxis(CAxis* axisDestination, CAxis* axisSource, CReduceAxisToScalar* algo)",
        << "Operation '" << op << "' not found. Please make sure to use a supported one"
 …
        << "Scalar destination " << scalarDestination->getId());
+  reduction_ = CReductionAlgorithm::createOperation((*CReductionAlgorithm::ReductionOperations_ptr)[op]);
+  //reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations[op]);
+  reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations_ptr->at(op));
+}
+void CScalarAlgorithmReduceAxis::apply(const std::vector<std::pair<int,double> >& localIndex,
+                                         const double* dataInput,
+                                         CArray<double,1>& dataOut,
+                                         std::vector<bool>& flagInitial,
+                                       bool ignoreMissingValue)
+void CScalarAlgorithmReduceAxis::apply(const std::vector<std::pair<int,double> >& localIndex, const double* dataInput, CArray<double,1>& dataOut,
+                                         std::vector<bool>& flagInitial, bool ignoreMissingValue, bool firstPass)
+{
   reduction_->apply(localIndex, dataInput, dataOut, flagInitial, ignoreMissingValue);
+  reduction_->apply(localIndex, dataInput, dataOut, flagInitial, ignoreMissingValue, firstPass);
+}
 …
   TransformationWeightMap& transWeight = this->transformationWeight_[0];
+  CArray<int,1>& axisSrcIndex = axisSrc_->index;
+  int globalIndexSize = axisSrcIndex.numElements();
+  int globalIndexSize = axisSrc_-> n_glo;
   for (int idx = 0; idx < globalIndexSize; ++idx)
+  {
     transMap[0].push_back(axisSrcIndex(idx));
+    transMap[0].push_back(idx);
     transWeight[0].push_back(1.0);
+  }

XIOS/dev/branch_openmp/src/transformation/scalar_algorithm_reduce_axis.hpp

-                      r1155
+                      r1328
 #include "scalar_algorithm_transformation.hpp"
 #include "transformation.hpp"
-#include "reduction.hpp"
 namespace xios {
 …
   Reducing an axis to a scalar
 */
 class CScalarAlgorithmReduceAxis : public CScalarAlgorithmTransformation, public CReductionAlgorithm
+class CScalarAlgorithmReduceAxis : public CScalarAlgorithmTransformation
+{
 public:
 …
                      CArray<double,1>& dataOut,
                      std::vector<bool>& flagInitial,
                      bool ignoreMissingValue);
+                     bool ignoreMissingValue, bool firstPass);
   virtual void updateData(CArray<double,1>& dataOut);

XIOS/dev/branch_openmp/src/transformation/scalar_algorithm_reduce_domain.cpp

-                      r1155
+                      r1328
 #include "grid_transformation_factory_impl.hpp"
+#include "reduction.hpp"
 namespace xios {
 …
+  }
+  // if(CReductionAlgorithm::ReductionOperations_ptr == 0)
+  //   CReductionAlgorithm::initReductionOperation();
+  if(CReductionAlgorithm::ReductionOperations_ptr == 0)
+    CReductionAlgorithm::initReductionOperation();
+  if ((*CReductionAlgorithm::ReductionOperations_ptr).end() == (*CReductionAlgorithm::ReductionOperations_ptr).find(op))
+  //if (CReductionAlgorithm::ReductionOperations.end() == CReductionAlgorithm::ReductionOperations.find(op))
+  if (CReductionAlgorithm::ReductionOperations_ptr->end() == CReductionAlgorithm::ReductionOperations_ptr->find(op))
     ERROR("CScalarAlgorithmReduceDomain::CScalarAlgorithmReduceDomain(CDomain* domainDestination, CDomain* domainSource, CReduceDomainToScalar* algo)",
        << "Operation '" << op << "' not found. Please make sure to use a supported one"
 …
        << "Scalar destination " << scalarDestination->getId());
+  reduction_ = CReductionAlgorithm::createOperation((*CReductionAlgorithm::ReductionOperations_ptr)[op]);
+  //reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations[op]);
+  reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations_ptr->at(op));
+}
 …
                                          CArray<double,1>& dataOut,
                                          std::vector<bool>& flagInitial,
                                          bool ignoreMissingValue)
+                                         bool ignoreMissingValue, bool firstPass)
+{
   reduction_->apply(localIndex, dataInput, dataOut, flagInitial, ignoreMissingValue);
+  reduction_->apply(localIndex, dataInput, dataOut, flagInitial, ignoreMissingValue, firstPass);
+}

XIOS/dev/branch_openmp/src/transformation/scalar_algorithm_reduce_domain.hpp

-                      r1155
+                      r1328
 #include "scalar_algorithm_transformation.hpp"
 #include "transformation.hpp"
-#include "reduction.hpp"
 namespace xios {
 …
   Reducing an DOMAIN to a scalar
 */
 class CScalarAlgorithmReduceDomain : public CScalarAlgorithmTransformation, public CReductionAlgorithm
+class CScalarAlgorithmReduceDomain : public CScalarAlgorithmTransformation
+{
 public:
 …
                      CArray<double,1>& dataOut,
                      std::vector<bool>& flagInitial,
                      bool ignoreMissingValue);
+                     bool ignoreMissingValue, bool firstPass);
   virtual void updateData(CArray<double,1>& dataOut);

XIOS/dev/branch_openmp/src/type/type.hpp

r1134	r1328
94	94	const CType_ref& operator = (CType<T>& val) const ;
95	95	const CType_ref& operator = (const CType_ref& val) const;
96		operator T&() const;
	96	operator T&() const;
97	97
98	98	inline virtual CBaseType* clone(void) const { return _clone(); }

XIOS/dev/branch_openmp/src/xios.hpp

-                      r591
+                      r1328
 /// XIOS headers ///
+#include "nc4_data_output.hpp"
+#include "data_output.hpp"
+//#include "nc4_data_output.hpp"
 using namespace xios;
 using namespace xios::xml;
 using namespace xios::func;
+//using namespace xios::xml;
+//using namespace xios::func;
 #endif //__XIOS__

XIOS/dev/branch_openmp/src/xios_server.f90

-                      r1134
+                      r1328
 PROGRAM server_main
   USE xios
-  USE mod_wait
   IMPLICIT NONE
   INCLUDE "mpif.h"
+  INTEGER :: ierr, th_level
+    CALL MPI_INIT(ierr)
+    !CALL MPI_INIT_thread(3, th_level, ierr)
+    CALL init_wait
+  INTEGER :: ierr
     CALL xios_init_server
-    CALL MPI_FINALIZE(ierr)
   END PROGRAM server_main

Context Navigation

Legend: