Add support for hdf5 VFD

CMakeLists.txt

@@ -36,6 +36,7 @@ option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to Tr
 option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF)
 option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF)
 option(PARTHENON_DISABLE_HDF5_COMPRESSION "HDF5 compression is enabled by default, set this to True to disable compression in HDF5 output/restart files" OFF)
+option(PARTHENON_ENABLE_HDF5_VFD "Use HDF5 Virtual File Driver (required for subfiling) and must be supported by the HDF5 lib. Disabled by default" OFF)
 option(PARTHENON_DISABLE_SPARSE "Sparse capability is enabled by default, set this to True to compile-time disable all sparse capability" OFF)
 option(PARTHENON_ENABLE_ASCENT "Enable Ascent for in situ visualization and analysis" OFF)
 option(PARTHENON_LINT_DEFAULT "Linting is turned off by default, use the \"lint\" target or set \
@@ -173,6 +174,10 @@ if (NOT PARTHENON_DISABLE_HDF5)
       "without compression can result in much larger HDF5 output and restart files as well as "
       "degraded I/O performance.")
     endif()
+
+    if (PARTHENON_ENABLE_HDF5_VFD AND (HDF5_VERSION VERSION_LESS "1.14.0"))
+      message(FATAL_ERROR "HDF5 VFD only supported with release later than 1.14.0")
+    endif()
   endif()
 
   # HDF5 Interface library

src/config.hpp.in

@@ -48,6 +48,15 @@
 // define PARTHENON_DISABLE_HDF5_COMPRESSION or not at all
 #cmakedefine PARTHENON_DISABLE_HDF5_COMPRESSION
 
+// define PARTHENON_ENABLE_HDF5_VFD or not at all
+// TODO(reviewers) is this appraoch that we want to adopt more generally?
+#cmakedefine01 PARTHENON_ENABLE_HDF5_VFD
+namespace parthenon {
+    struct BuildOptions {
+        constexpr static bool hdf5_vfd = static_cast<bool>(PARTHENON_ENABLE_HDF5_VFD);
+    };
+}
+
 // define ENABLE_SPARSE or not at all
 #cmakedefine ENABLE_SPARSE
 

src/parthenon_manager.cpp

@@ -52,10 +52,27 @@ ParthenonStatus ParthenonManager::ParthenonInit(int argc, char *argv[]) {
 ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) {
   // initialize MPI
 #ifdef MPI_PARALLEL
-  if (MPI_SUCCESS != MPI_Init(&argc, &argv)) {
-    std::cout << "### FATAL ERROR in ParthenonInit" << std::endl
-              << "MPI Initialization failed." << std::endl;
-    return ParthenonStatus::error;
+  if constexpr (BuildOptions::hdf5_vfd) {
+    int mpi_thread_required = MPI_THREAD_MULTIPLE;
+    int mpi_thread_provided = 0;
+
+    if (MPI_SUCCESS !=
+        MPI_Init_thread(&argc, &argv, mpi_thread_required, &mpi_thread_provided)) {
+      std::cout << "### FATAL ERROR in ParthenonInit" << std::endl
+                << "MPI Initialization failed." << std::endl;
+      return ParthenonStatus::error;
+    }
+    if (mpi_thread_provided < mpi_thread_required) {
+      std::cout << "### FATAL ERROR in ParthenonInit" << std::endl
+                << "Provided MPI thread level not enough." << std::endl;
+      return ParthenonStatus::error;
+    }
+  } else {
+    if (MPI_SUCCESS != MPI_Init(&argc, &argv)) {
+      std::cout << "### FATAL ERROR in ParthenonInit" << std::endl
+                << "MPI Initialization failed." << std::endl;
+      return ParthenonStatus::error;
+    }
   }
   // Get process id (rank) in MPI_COMM_WORLD
   if (MPI_SUCCESS != MPI_Comm_rank(MPI_COMM_WORLD, &(Globals::my_rank))) {