Asynchronous I/O support & a new Data rearranger

.github/workflows/spio_pnetcdf_testing_workflow.yaml

@@ -19,6 +19,9 @@ on:  # yamllint disable-line rule:truthy
   push:
     branches:
       - master
+      # Includes all branches starting with "copilot/"
+      # A "*" matches any char except "/", so use "**" to handle all branch names
+      - 'copilot/**'
   pull_request:
     branches:
       - master

AGENTS.md

@@ -0,0 +1,49 @@
+# AGENTS.md
+
+SCORPIO (Software for Caching Output and Reads for Parallel I/O) is a high-level Parallel I/O Library for structured grid applications.
+
+## Project Overview
+
+SCORPIO (Software for Caching Output and Reads for Parallel I/O) is a high-level Parallel I/O Library. The library is used by the E3SM (Energy Exascale Earth System Model) earth system model for all I/O.
+
+- **Website:** https://github.com/E3SM-Project/scorpio
+- **Documentation:** https://docs.e3sm.org/scorpio/
+- **DOI:** https://www.osti.gov/doecode/biblio/36752
+
+## Architecture
+
+- The Fortran interface library source is in src/flib directory
+- The C (main/core library) library source is in src/clib directory
+- The tests are in tests/general and tests/cunit directories
+- The examples are in the examples directory
+
+## Build Commands
+
+SCORPIO uses CMake for configuring the library. The library requires a C, C++ and Fortran compiler and an MPI library. SCORPIO uses low-level I/O libraries like PnetCDF (https://parallel-netcdf.github.io), NetCDF (https://www.unidata.ucar.edu/software/netcdf), HDF5 (https://www.hdfgroup.org/solutions/hdf5/) and ADIOS (https://adios2.readthedocs.io/en/latest/) for I/O.
+
+To configure SCORPIO (source in /path/to/scorpio/source directory) using CMake with the PnetCDF library (installed at /path/to/pnetcdf) use the command below,
+
+```
+CC=mpicc CXX=mpicxx FC=mpif90 cmake -DPnetCDF_PATH=/path/to/pnetcdf /path/to/scorpio/source
+```
+
+After configuring the library to build it use make,
+
+```
+make
+```
+
+## Development Workflow
+- All changes are made on a development branch off master
+- The development branches are named using the pattern : ```<github username>/<feature description>```
+- The changes in a branch are merged to the develop branch for nightly testing (Run using Jenkins and results published on CDASH at http://my.cdash.org/index.php?project=E3SM\_SCORPIO)
+- Once the nightly testing is successful the PR is manually merged to the master branch
+- Code formatting for C++ sources is mentioned in the library documentation (https://docs.e3sm.org/scorpio/html/contributing_code.html)
+
+## Testing instructions
+- Find the CI (Github Actions workflow) plan in the .github/workflows directory
+- Add/update tests for the code that you change
+
+## PR instructions
+- Run the CI workflow (SCORPIO + PnetCDF - build and test) in the .github/workflows directory before committing any changes
+- Do not automatically merge PRs to master or develop branch

AUTHORS.md

@@ -0,0 +1,14 @@
+# Authors
+** Current Authors **
+
+* Jayesh Krishna [email](jayesh@anl.gov)
+* Danqing Wu [email](DanqingWu@hotmail.com)
+
+** Past Contributors **
+
+* Norbert Podhorszki
+* Tahsin Kurc
+* Dmitry Ganyushin
+* Jim Edwards
+* Ed Hartnett
+

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required (VERSION 3.7)
+cmake_minimum_required (VERSION 3.7...4.0.1)
 project (SCORPIO C CXX Fortran)
 #cmake_policy(VERSION 3.5.2)
 
@@ -49,6 +49,7 @@ option (PIO_USE_MALLOC       "Use native malloc (instead of bget package)"  OFF)
 option (PIO_USE_INDEP_MODE   "Use PnetCDF independent data mode"            ON)
 option (PIO_MICRO_TIMING     "Enable internal micro timers"                 OFF)
 option (PIO_SAVE_DECOMPS     "Dump the decomposition information"           OFF)
+option (PIO_USE_ASYNC_WR_THREAD    "Use an asynchronous thread for writes (Requires C++11 support)" OFF)
 option (PIO_LIMIT_CACHED_IO_REGIONS  "Limit the number of non-contiguous regions in an IO process" OFF)
 option (WITH_PNETCDF         "Require the use of PnetCDF"                   ON)
 option (WITH_NETCDF          "Require the use of NetCDF"                    ON)
@@ -158,6 +159,12 @@ else()
   message (STATUS "Disabling saving I/O decompositions (default)")
 endif()
 
+set(USE_ASYNC_WR_THREAD 0)
+if(PIO_USE_ASYNC_WR_THREAD)
+  message(STATUS "Using asynchronous I/O thread for writes")
+  set(USE_ASYNC_WR_THREAD 1)
+endif()
+
 if(PIO_LIMIT_CACHED_IO_REGIONS)
   set(LIMIT_CACHED_IO_REGIONS 1)
   if(NOT DEFINED PIO_MAX_CACHED_IO_REGIONS)
@@ -211,6 +218,14 @@ else()
   endif()
 endif()
 
+# Default - Do not disable MPI file syncing for HDF5 output files
+set(SPIO_DISABLE_HDF5_MPI_FILE_SYNC 0)
+if(FQDN_SITENAME MATCHES "^compute-386")
+  # On ANL compute nodes with NFS+ROMIO, disable MPI file syncing for HDF5 output
+  message(STATUS "Disabling HDF5 file syncing")
+  set(SPIO_DISABLE_HDF5_MPI_FILE_SYNC 1)
+endif()
+
 set(USE_INDEP_MODE 0)
 if(PIO_USE_INDEP_MODE)
   if(FQDN_SITENAME MATCHES "^chrlogin" OR FQDN_SITENAME MATCHES "^compute-240" OR FQDN_SITENAME MATCHES "^compute-386")
@@ -251,6 +266,28 @@ else()
   message(STATUS "Setting chunk size (in bytes) for HDF5/PnetCDF chunked variables, requested bytes = " ${PIO_CHUNK_SIZE} " (default)")
 endif()
 
+# Expected format for SPIO_DIM_CHUNK_INFO = "DIM1_NAME:DIM1_CHUNK_SZ;DIM2_NAME:DIM2_CHUNK_SZ;"
+# e.g. "time:1;lev:8;"   ---> Each chunk has 1 time dimension (time) and 8 levels (lev)
+if(DEFINED SPIO_DIM_CHUNK_INFO)
+  message(STATUS "Using user-specified dimension chunk info, " ${SPIO_DIM_CHUNK_INFO})
+endif()
+
+# Add more constraints on where chunking is enabled. Specify a regex that matches the variable and file name where chunking
+# is enabled. Chunking will be disabled for all other variables
+# The regex is same as the one used to save decompositions
+# e.g. -DSPIO_ENABLE_CHUNKING_REGEX:STRING='(VAR=\".*U.*\")&&(FILE=\".*e3sm_fgi.*\")'
+#       Chunking is enabled only on variables with names that match ".*U.*" in files with names that match ".*e3sm_fgi.*"
+if(DEFINED SPIO_ENABLE_CHUNKING_REGEX)
+  if(DEFINED SPIO_DIM_CHUNK_INFO)
+    message(STATUS "Using user-specified regex," ${SPIO_ENABLE_CHUNKING_REGEX} " to decide when to enable chunking of variables")
+  else()
+    message(STATUS "WARNING: SPIO_ENABLE_CHUNKING_REGEX was defined but SPIO_DIM_CHUNK_INFO was not defined, disabling SPIO_ENABLE_CHUNKING_REGEX")
+    set(SPIO_ENABLE_CHUNKING_REGEX "*")
+  endif()
+else()
+  set(SPIO_ENABLE_CHUNKING_REGEX "*")
+endif()
+
 if(WITH_ADIOS2)
   # Maximum number of I/O decompositions registered with ADIOS type
   set(DEF_SPIO_MAX_ADIOS_DECOMPS 65536)
@@ -325,6 +362,32 @@ else()
   endif()
 endif()
 
+# Asynchronous I/O options
+# Maximum wait for asynchronous I/O operations (in milliseconds)
+if(NOT DEFINED SPIO_ASYNC_OP_WAIT_TIMEOUT)
+  # 2 minutes = 2 * 60 * 1000 milliseconds
+  set(SPIO_ASYNC_OP_WAIT_TIMEOUT 120000)
+  if(PIO_USE_ASYNC_WR_THREAD)
+    message(STATUS "Setting timeout for asynchronous I/O operations to ${SPIO_ASYNC_OP_WAIT_TIMEOUT} ms (default)")
+  endif()
+else()
+  if(PIO_USE_ASYNC_WR_THREAD)
+    message(STATUS "Setting timeout for asynchronous I/O operations to ${SPIO_ASYNC_OP_WAIT_TIMEOUT} ms")
+  endif()
+endif()
+
+# Maximum number of threads performing asynchronous I/O operations
+if(NOT DEFINED SPIO_ASYNC_NTHREADS)
+  set(SPIO_ASYNC_NTHREADS 1)
+  if(PIO_USE_ASYNC_WR_THREAD)
+    message(STATUS "Setting number of asynchronous I/O threads to ${SPIO_ASYNC_NTHREADS} (default)")
+  endif()
+else()
+  if(PIO_USE_ASYNC_WR_THREAD)
+    message(STATUS "Setting number of asynchronous I/O threads to ${SPIO_ASYNC_NTHREADS}")
+  endif()
+endif()
+
 if(DEFINED SPIO_OVERRIDE_HDF5_COMPRESSION_VNAME_REGEX)
   if(WITH_HDF5 AND HDF5_USE_COMPRESSION)
     message(STATUS "Overriding HDF5 compression for variables matching regex: ${SPIO_OVERRIDE_HDF5_COMPRESSION_VNAME_REGEX}")

cmake/SPIOUtils.cmake

@@ -9,7 +9,8 @@
 # Prereq: MPI (MPI serial) & GPTL libraries are searched for (find_library() )
 # prior to invoking this macro to add SCORPIO applications
 macro (add_spio_executable EXE_NAME IS_C_SRC EXE_LINKER_LANGUAGE)
-  add_executable(${EXE_NAME} ${ARGN})
+  cmake_parse_arguments (_SPIO_EXE "" "" "OBJ_LIBS" ${ARGN})
+  add_executable(${EXE_NAME} ${_SPIO_EXE_UNPARSED_ARGUMENTS})
   if (${IS_C_SRC})
     target_link_libraries(${EXE_NAME} PRIVATE pioc)
   else ()
@@ -47,5 +48,10 @@ macro (add_spio_executable EXE_NAME IS_C_SRC EXE_LINKER_LANGUAGE)
   if (NOT ("${EXE_LINKER_LANGUAGE}" STREQUAL ""))
     set_property(TARGET ${EXE_NAME} PROPERTY LINKER_LANGUAGE ${EXE_LINKER_LANGUAGE})
   endif ()
+  if (_SPIO_EXE_OBJ_LIBS)
+    foreach (_spio_obj_lib ${_SPIO_EXE_OBJ_LIBS})
+      target_link_libraries (${EXE_NAME} PRIVATE $<TARGET_OBJECTS:${_spio_obj_lib}>)
+    endforeach ()
+  endif ()
 endmacro ()
 

examples/c/CMakeLists.txt

@@ -72,6 +72,9 @@ endif ()
 #==============================================================================
 #  DEFINE THE TARGETS AND TESTS
 #==============================================================================
+# Test timeout to 180s
+set (DEFAULT_TEST_TIMEOUT 180)
+
 add_spio_executable(examplePio TRUE "" examplePio.c)
 add_spio_executable(example1 TRUE "" example1.c)
 add_spio_executable(darray_no_async TRUE "" darray_no_async.c)
@@ -101,18 +104,18 @@ if (PIO_USE_MPISERIAL)
   add_test(NAME example1 COMMAND example1)
   add_test(NAME put_var COMMAND put_var)
 else ()
-  add_mpi_test(examplePio EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/examplePio NUMPROCS 4 TIMEOUT 60)
-  add_mpi_test(example1 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/example1 NUMPROCS 4 TIMEOUT 60)
-  add_mpi_test(put_var EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/put_var NUMPROCS 4 TIMEOUT 60)
-  #add_mpi_test(darray_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_async NUMPROCS 5 TIMEOUT 60)
-  add_mpi_test(darray_no_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_no_async NUMPROCS 4 TIMEOUT 60)
+  add_mpi_test(examplePio EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/examplePio NUMPROCS 4 TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(example1 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/example1 NUMPROCS 4 TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(put_var EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/put_var NUMPROCS 4 TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  #add_mpi_test(darray_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_async NUMPROCS 5 TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(darray_no_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_no_async NUMPROCS 4 TIMEOUT ${DEFAULT_TEST_TIMEOUT})
   if (WITH_HDF5)
-    add_mpi_test(test_hdf5 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_hdf5 NUMPROCS 4 TIMEOUT 60)
+    add_mpi_test(test_hdf5 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_hdf5 NUMPROCS 4 TIMEOUT ${DEFAULT_TEST_TIMEOUT})
   endif ()
   if (WITH_ADIOS2)
-    add_mpi_test(test_adios EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_adios NUMPROCS 4 TIMEOUT 60)
+    add_mpi_test(test_adios EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_adios NUMPROCS 4 TIMEOUT ${DEFAULT_TEST_TIMEOUT})
   endif ()
   if (WITH_NETCDF)
-    add_mpi_test(test_netcdf4p EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_netcdf4p NUMPROCS 4 TIMEOUT 60)
+    add_mpi_test(test_netcdf4p EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_netcdf4p NUMPROCS 4 TIMEOUT ${DEFAULT_TEST_TIMEOUT})
   endif ()
 endif ()

examples/cxx/e3sm_fgi.cpp

@@ -154,9 +154,13 @@ static int get_user_options(
 
 int main(int argc, char *argv[])
 {
-  int ret = 0, rank = 0;
+  int ret = 0, rank = 0, tmode;
 
+#if PIO_USE_ASYNC_WR_THREAD
+  MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &tmode);
+#else
   MPI_Init(&argc, &argv);
+#endif
 
   Util::GVars::logger = Util::Logging::Logger::get_logger(MPI_COMM_WORLD, "STDOUT");
   Util::GVars::logger->set_log_level(Util::Logging::LogLevel::STATUS);