Merge branch 'development' of https://github.com/dash-project/dash into feat-halo

Author: dhinf

CMakeExt/GenerateDASHCXX.cmake

@@ -39,10 +39,16 @@ if (";${DART_IMPLEMENTATIONS_LIST};" MATCHES ";mpi;")
     set(ADDITIONAL_LIBRARIES_WRAP
         "${ADDITIONAL_LIBRARIES_WRAP} ${MPI_C_LIB}")
   endforeach()
-  set(ADDITIONAL_INCLUDES_WRAP "${ADDITIONAL_INCLUDES_WRAP} -I${MPI_INCLUDE_PATH}")
+  foreach (ADDITIONAL_INCLUDE_PATH ${MPI_INCLUDE_PATH})
+    set(ADDITIONAL_INCLUDES_WRAP
+        "${ADDITIONAL_INCLUDES_WRAP} -I${ADDITIONAL_INCLUDE_PATH}")
+  endforeach()
   set(DASHCC ${CMAKE_CXX_COMPILER})
   set(DART_IMPLEMENTATION "mpi")
-  set(ADDITIONAL_COMPILE_FLAGS_WRAP "${ADDITIONAL_COMPILE_FLAGS_WRAP} ${MPI_ADDITIONAL_COMPILE_FLAGS}")
+  foreach (ADDITIONAL_COMPILE_FLAG ${MPI_ADDITIONAL_COMPILE_FLAGS})
+    set(ADDITIONAL_COMPILE_FLAGS_WRAP
+        "${ADDITIONAL_COMPILE_FLAGS_WRAP} ${ADDITIONAL_COMPILE_FLAG}")
+  endforeach()
   configure_file(
     ${CMAKE_SOURCE_DIR}/dash/scripts/dashcc/dashcxx.in
     ${CMAKE_BINARY_DIR}/bin/dash-mpicxx

dart-impl/base/CMakeLists.txt

@@ -112,7 +112,7 @@ target_link_libraries(
 # Compile flags
 set_target_properties(
   ${DASH_DART_BASE_LIBRARY} PROPERTIES
-  COMPILE_FLAGS ${ADDITIONAL_COMPILE_FLAGS}
+  COMPILE_FLAGS "${ADDITIONAL_COMPILE_FLAGS}"
   C_STANDARD ${DART_C_STD_PREFERED}
   C_STANDARD_REQUIRED ON
   POSITION_INDEPENDENT_CODE TRUE

dart-impl/mpi/CMakeLists.txt

@@ -213,7 +213,7 @@ endif()
 # Compile flags
 set_target_properties(
   ${DASH_DART_IMPL_MPI_LIBRARY} PROPERTIES
-  COMPILE_FLAGS ${ADDITIONAL_COMPILE_FLAGS_STR}
+  COMPILE_FLAGS "${ADDITIONAL_COMPILE_FLAGS_STR}"
   C_STANDARD ${DART_C_STD_PREFERED}
   C_STANDARD_REQUIRED ON
 )

dart-impl/mpi/src/dart_communication.c

@@ -149,14 +149,14 @@ static dart_ret_t put_shared_mem(
     size_t                      nelem,
     dart_datatype_t             dtype)
 {
-  DART_LOG_DEBUG("dart_get: using shared memory window in segment %d enabled",
+  DART_LOG_DEBUG("dart_put: using shared memory window in segment %d enabled",
       seginfo->segid);
   dart_team_unit_t luid    = team_data->sharedmem_tab[unitid.id];
   char *           baseptr = seginfo->baseptr[luid.id];
 
   baseptr += offset;
   DART_LOG_DEBUG(
-      "dart_get: memcpy %zu bytes", nelem * dart__mpi__datatype_sizeof(dtype));
+      "dart_put: memcpy %zu bytes", nelem * dart__mpi__datatype_sizeof(dtype));
   memcpy(baseptr, src, nelem * dart__mpi__datatype_sizeof(dtype));
   return DART_OK;
 }

dart-impl/shmem/CMakeLists.txt

@@ -66,7 +66,7 @@ target_link_libraries(
 
 set_target_properties(
   ${DASH_DART_IMPL_SHMEM_LIBRARY} PROPERTIES
-  COMPILE_FLAGS ${ADDITIONAL_COMPILE_FLAGS}
+  COMPILE_FLAGS "${ADDITIONAL_COMPILE_FLAGS}"
 )
 
 DeployLibrary(${DASH_DART_IMPL_SHMEM_LIBRARY})

dash/CMakeLists.txt

@@ -338,7 +338,7 @@ foreach (dart_variant ${DART_IMPLEMENTATIONS_LIST})
   )
   set_target_properties(
     ${DASH_LIBRARY} PROPERTIES
-    CXX_STANDARD ${DASH_CXX_STD_PREFERED}
+    CXX_STANDARD "${DASH_CXX_STD_PREFERED}"
     CXX_STANDARD_REQUIRED ON
   )
 

dash/include/dash/io/hdf5/StorageDriver.h

@@ -544,12 +544,13 @@ class StoreHDF {
   }
 
   /**
-   * Convert a dash pattern into a hdf5 pattern in form of a list of hdf5 slabs.
+   * Convert a dash pattern into a hdf5 pattern in form of a list of hdf5 slabs and 
+   * is used for Patterns supporting underfilled blocks
    */
-  template <dim_t ndim, MemArrange Arr, typename index_t>
-  const static inline std::vector<hdf5_hyperslab_spec<ndim>>
-      /// The dash pattern to convert to hdf5 slabs
-      _get_hdf_slabs(const dash::Pattern<ndim, Arr, index_t>& pattern) {
+  template <typename PatternT>
+  const static inline std::vector<hdf5_hyperslab_spec<PatternT::ndim()>>
+  _get_hdf_slabs_with_underfilled(const PatternT& pattern) {
+    static constexpr auto ndim = PatternT::ndim();
     std::vector<hdf5_hyperslab_spec<ndim>> specs_hyperslab;
     std::vector<dim_t> dimensions;
 
@@ -575,9 +576,37 @@ class StoreHDF {
     return specs_hyperslab;
   }
 
+   /**
+   * Convert a dash pattern into a hdf5 pattern in form of a list of hdf5 slabs.
+   * Spezialization for BlockPattern
+   */
+  template <dim_t ndim, MemArrange Arr, typename index_t>
+  const static inline std::vector<hdf5_hyperslab_spec<ndim>>
+  /// The dash pattern to convert to hdf5 slabs
+  _get_hdf_slabs(const dash::Pattern<ndim, Arr, index_t>& pattern) {
+
+    return _get_hdf_slabs_with_underfilled(pattern);
+  }
+
+  /**
+   * Convert a dash pattern into a hdf5 pattern in form of a list of hdf5 slabs.
+   * Spezialization for TilePattern
+   */
+  template <dim_t ndim, MemArrange Arr, typename index_t>
+  const static inline std::vector<hdf5_hyperslab_spec<ndim>>
+  /// The dash pattern to convert to hdf5 slabs
+  _get_hdf_slabs(const dash::TilePattern<ndim, Arr, index_t>& pattern) {
+
+    return _get_hdf_slabs_with_underfilled(pattern);
+  }
+
+  /**
+   * Convert a dash pattern into a hdf5 pattern in form of a list of hdf5 slabs.
+   */
   template <class pattern_t>
   const static inline std::vector<hdf5_hyperslab_spec<pattern_t::ndim()>>
   _get_hdf_slabs(const pattern_t& pattern) {
+
     return std::vector<hdf5_hyperslab_spec<pattern_t::ndim()>>(
         1, _get_hdf_slab_body(pattern, {}));
   }

dash/include/dash/pattern/TilePattern.h

@@ -1240,6 +1240,7 @@ class TilePattern
                    "unit:",       unit,
                    "lblock_idx:", local_block_index,
                    "lblockspec:", _local_blockspec.extents());
+
     // Local block index to local block coords:
     auto l_block_coords = _local_blockspec.coords(local_block_index);
     auto unit_ts_coords = _teamspec.coords(unit);
@@ -1248,10 +1249,20 @@ class TilePattern
     std::array<size_type, NumDimensions>  extents{};
     for (auto d = 0; d < NumDimensions; ++d) {
       auto blocksize_d = _blocksize_spec.extent(d);
-      auto nunits_d    = _teamspec.extent(d);
+
       // Block offsets are lobal coordinates of first block element:
+      auto nunits_d    = _teamspec.extent(d);
       offsets[d] = ((l_block_coords[d] * nunits_d) + unit_ts_coords[d]) *
                    blocksize_d;
+      // checks if the local block is the last one in this dimension
+      auto num_blocks_d =_local_blockspec.extent(d);
+      if(l_block_coords[d] == (num_blocks_d - 1)){
+          size_type remaining = local_extent(d) % blocksize_d;
+          // if block is underfilled the extent will be decreased
+          if(remaining > 0) {
+            blocksize_d = remaining;
+          }
+      }
       extents[d] = blocksize_d;
     }
     ViewSpec_t block_vs(offsets, extents);
@@ -1484,6 +1495,26 @@ class TilePattern
     return NumDimensions;
   }
 
+    /**
+   * Number of elements missing in the overflow block of given dimension
+   * compared to the regular blocksize (\see blocksize(d)), with
+   * 0 <= \c underfilled_blocksize(d) < blocksize(d).
+   */
+  SizeType underfilled_blocksize(dim_t dimension) const
+  {
+    // Underflow blocksize = regular blocksize - overflow blocksize:
+    auto ovf_blocksize = _memory_layout.extent(dimension) %
+                         blocksize(dimension);
+
+    if (ovf_blocksize == 0) {
+      return 0;
+    }
+
+    DASH_LOG_DEBUG_VAR("TilePattern.underfilled_blocksize", ovf_blocksize);
+    auto reg_blocksize = blocksize(dimension);
+    return reg_blocksize - ovf_blocksize;
+  }
+
 private:
 
   TilePattern(const PatternArguments_t & arguments)
@@ -1543,11 +1574,7 @@ class TilePattern
                            extent_d, // size of range (extent)
                            units_d   // number of blocks (units)
                          );
-      DASH_ASSERT_EQ(0, extent_d % blocksize_d,
-                     "TilePattern requires balanced block sizes: " <<
-                     "extent "    << extent_d    << " is no multiple of " <<
-                     "block size" << blocksize_d << " in " <<
-                     "dimension " << d);
+
       s_blocks[d] = blocksize_d;
     }
     DASH_LOG_TRACE_VAR("TilePattern.init_blocksizespec >", s_blocks);
@@ -1628,6 +1655,7 @@ class TilePattern
       l_blocks[d] = num_l_blocks_d;
     }
     DASH_LOG_TRACE_VAR("TilePattern.init_local_blockspec >", l_blocks);
+    DASH_LOG_DEBUG_VAR("TilePattern.init_local_extents >", initialize_local_extents(unit_id));
     return BlockSpec_t(l_blocks);
   }
 
@@ -1698,9 +1726,25 @@ class TilePattern
       if (num_odd_blocks_d > unit_ts_coords[d]) {
         ++num_l_blocks_d;
       }
+      
+      // Coordinate of this unit id in teamspec in dimension:
+      auto unit_ts_coord      = unit_ts_coords[d];
       // Possibly there are more blocks than units in dimension and no
       // block left for this unit. Local extent in d then becomes 0.
       l_extents[d] = num_l_blocks_d * blocksize_d;
+      // Unit id assigned to the last block in dimension:
+      team_unit_t last_block_unit_d((num_blocks_d % num_units_d == 0)
+                                      ? num_units_d - 1
+                                      : (num_blocks_d % num_units_d) - 1);
+
+      if(unit_ts_coord == last_block_unit_d) {
+        // If the last block in the dimension is underfilled and
+        // assigned to the local unit, subtract the missing extent:
+        auto undfill_blocksize_d = underfilled_blocksize(d);
+        DASH_LOG_TRACE_VAR("TilePattern.init_local_extents",
+                           undfill_blocksize_d);
+        l_extents[d] -= undfill_blocksize_d;
+      }
     }
     DASH_LOG_DEBUG_VAR("TilePattern.init_local_extents >", l_extents);
     return l_extents;

dash/test/io/HDF5MatrixTest.cc

@@ -277,6 +277,61 @@ TEST_F(HDF5MatrixTest, UnderfilledPattern) {
   verify_matrix(matrix_b, 1);
 }
 
+template<typename MatrixT>
+void print_matrix(const MatrixT& matrix) {
+  using namespace std;
+  dash::barrier();
+  if(!dash::myid()) {
+    auto rows = matrix.extent(0);
+    auto cols = matrix.extent(1);
+    cout << fixed;
+    cout.precision(4);
+    cout << "Matrix:" << endl;
+    for (auto r = 0; r < rows; ++r) {
+      for (auto c = 0; c < cols; ++c) {
+        auto lindex = matrix.pattern().local_index({r,c});
+        cout << " " << setw(3) << (double) matrix[r][c] << "(" << lindex.unit << "," << lindex.index << ")";
+      }
+      cout << endl;
+    }
+  }
+  dash::barrier();
+}
+
+/**
+ * Allocate a matrix with extents that cannot fit into full blocks
+ * TilePattern
+ */
+TEST_F(HDF5MatrixTest, UnderfilledPatternTile) {
+  using Matrix_t = dash::Matrix<double, 2>;
+
+  auto team_size = dash::Team::All().size();
+
+  int ext_x = 5 * team_size + 1;
+  int ext_y = 10 * team_size;
+
+  double test_value = dash::myid()+1;
+
+  LOG_MESSAGE("Matrix extent (%i,%i)", ext_x, ext_y);
+
+  auto size_spec = dash::SizeSpec<2>(ext_x, ext_y);
+
+  {
+    Matrix_t matrix_a(size_spec);
+    //dash::fill(matrix_a.begin(), matrix_a.end(), test_value);
+    fill_matrix(matrix_a, test_value);
+    dio::OutputStream os(_filename);
+    os << dio::dataset(_dataset) << matrix_a;
+  }
+
+  Matrix_t matrix_b(size_spec);
+  dio::InputStream is(_filename);
+  is >> dio::dataset(_dataset) >> matrix_b;
+  dash::barrier();
+
+  verify_matrix(matrix_b, test_value);
+}
+
 TEST_F(HDF5MatrixTest, UnderfilledPatMultiple) {
   typedef dash::Pattern<2, dash::ROW_MAJOR> pattern_t;
   typedef typename pattern_t::index_type index_t;
@@ -519,23 +574,23 @@ TEST_F(HDF5MatrixTest, DashView)
   int secret = 0;
   dash::NArray<int,2> matrix2d(100,80);
   fill_matrix(matrix2d, secret);
-  
+
   {
     auto output_view = dash::sub(0,5, matrix2d);
-    
+
     // TODO: Does not work
     dash::fill(dash::begin(output_view), dash::end(output_view), 5);
 
     // Store View
     dio::OutputStream os(_filename);
     os << output_view;
   }
-  
+
   // Load data into a existing view
   auto input_view = dash::sub(0,10, matrix2d);
   dio::InputStream is(_filename);
   is >> input_view;
-  
+
   // Load data into new matrix
   dash::Array<int> matrix1d;
   is >> matrix1d;
@@ -548,20 +603,20 @@ TEST_F(HDF5MatrixTest, HDFView)
   int secret = 0;
   dash::NArray<int,3> matrix3d(10,8,5);
   fill_matrix(matrix3d, secret);
-  
+
   {
     dio::OutputStream os(_filename);
     os << matrix3d;
   }
-  
+
   // Load subset of data into new matrix
   dash::NArray<int,3> sub_matrix3d;
   auto position = std::array<size_t,3>({2,4,1}); // top left corner of desired block
   auto extent   = std::array<size_t,3>({4,2,1}); // extents of block
   dio::InputStream is(_filename);
   is >> dash::block(position, extent)
      >> sub_matrix3d;
-  
+
   // post condition
   // sub_matrix3d has extent {4,2,1}
 }
@@ -571,29 +626,29 @@ TEST_F(HDF5MatrixTest, WriteCorresponding)
   int secret = 0;
   dash::NArray<int,3> matrix3d(10,8,5);
   fill_matrix(matrix3d, secret);
-  
+
   {
     dio::OutputStream os(_filename);
     os << matrix3d;
   }
-  
+
   auto view = matrix3d.col(1);
   // modify view
   dash::fill(view.begin(), view.end(), 0);
-  
+
   // write slice back
   dio::OutputStream os(_filename, dio::DeviceMode::App);
   is >> dio::modify_dataset()
      >> dash::block(position, extent)
      >> sub_matrix3d;
-  
+
   // post condition
   // slice in file is updated
   dash::NArray<int,3> new_matrix3d(10,8,5);
-  
+
   dio::InputStream is(_filename);
   is << new_matrix3d;
-  
+
   auto view = new_matrix3d.col(1);
   dash::for_each(view.begin(), view.end(), [](int & el){ASSERT_EQ_U(0, el)});
 }