Catalyst support (#4011)

## Summary This PR completes the work of @andrewcombs and @c-wetterer-nelson from #3444 by adding a docker image that includes ParaView and can be used for testing. It also adds a minor fix a02c1a5 discovered when testing these changes with the current master of WarpX. ## Additional background Supersedes and closes #3444 ## Checklist The proposed changes: - [x] fix a bug or incorrect behavior in AMReX - [x] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate --------- Co-authored-by: Andrew Combs <[email protected]> Co-authored-by: Corey Wetterer-Nelson <[email protected]>
AMReX-Codes · Jul 30, 2024 · 83ecf62 · 83ecf62
1 parent a8c5255
commit 83ecf62
Show file tree

Hide file tree

Showing 13 changed files with 108 additions and 17 deletions.
diff --git a/.github/workflows/catalyst.yml b/.github/workflows/catalyst.yml
@@ -0,0 +1,33 @@
+name: 🐧 Catalyst
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.head_ref }}-insituvis-catalyst
+  cancel-in-progress: true
+
+jobs:
+  catalyst:
+    name: Catalyst
+    runs-on: ubuntu-22.04
+    if: github.event.pull_request.draft == false
+    env:
+      CXX: g++
+      CC: gcc
+      CMAKE_PREFIX: "/opt/conduit;/opt/catalyst"
+    container:
+      image: kitware/paraview:ci-catalyst-amrex-warpx-20240701
+    steps:
+    - uses: actions/checkout@v4
+    - name: Configure
+      run: |
+        cmake -S . -B build          \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DAMReX_ENABLE_TESTS=ON  \
+            -DAMReX_FORTRAN=OFF      \
+            -DAMReX_CATALYST=ON      \
+            -DAMReX_CONDUIT=ON
+    - name: Build
+      run: |
+        cmake --build build -j 4
+
diff --git a/Docs/sphinx_documentation/source/BuildingAMReX.rst b/Docs/sphinx_documentation/source/BuildingAMReX.rst
@@ -508,6 +508,8 @@ The list of available options is reported in the :ref:`table <tab:cmakevar>` bel
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_CONDUIT                |  Enable Conduit support                         | NO                      | YES, NO               |
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
+   | AMReX_CATALYST               |  Enable Catalyst support                        | NO                      | YES, NO               |
+   +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_ASCENT                 |  Enable Ascent support                          | NO                      | YES, NO               |
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_HYPRE                  |  Enable HYPRE interfaces                        | NO                      | YES, NO               |

diff --git a/Src/CMakeLists.txt b/Src/CMakeLists.txt
@@ -43,7 +43,6 @@ include(AMReXParallelBackends)
 # Add definitions
 #
 include(AMReXSetDefines)
-
 #
 # Find and link third party libraries if needed
 #
@@ -188,7 +187,7 @@ if (AMReX_SENSEI)
    add_subdirectory(Extern/SENSEI)
 endif ()
 
-if (AMReX_CONDUIT OR AMReX_ASCENT)
+if (AMReX_CONDUIT OR AMReX_ASCENT OR AMReX_CATALYST)
    add_subdirectory(Extern/Conduit)
 endif ()
 

diff --git a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H
@@ -172,25 +172,30 @@ ParticleTileToBlueprint(const ParticleTile<ParticleType,
     } else {
         const auto &soa = ptile.GetStructOfArrays();
 
-        // for soa entries, we can use standard strides,
-        // since these are contiguous arrays
 
-        // id is the first int entry
-        conduit::Node &n_f_id = n_fields[topology_name + "_id"];
+        if(!soa.GetIntData().empty())
+        {
 
-        n_f_id["topology"] = topology_name;
-        n_f_id["association"] = "element";
-        n_f_id["values"].set_external(const_cast<int*>(&soa.GetIntData(0)[0]),
-                                      num_particles);
+            // for soa entries, we can use standard strides,
+            // since these are contiguous arrays
 
-        // cpu is the second int entry
-        conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"];
+            // id is the first int entry
+            conduit::Node &n_f_id = n_fields[topology_name + "_id"];
 
-        n_f_cpu["topology"] = topology_name;
-        n_f_cpu["association"] = "element";
-        n_f_cpu["values"].set_external(const_cast<int*>(&soa.GetIntData(0)[0]),
-                                       num_particles);
+            n_f_id["topology"] = topology_name;
+            n_f_id["association"] = "element";
+            n_f_id["values"].set_external(const_cast<int*>(&soa.GetIntData(0)[0]),
+                                          num_particles);
 
+            // cpu is the second int entry
+            conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"];
+
+            n_f_cpu["topology"] = topology_name;
+            n_f_cpu["association"] = "element";
+            n_f_cpu["values"].set_external(const_cast<int*>(&soa.GetIntData(0)[0]),
+                                           num_particles);
+
+        }
     }
 
     // --------------------------------

diff --git a/Src/Extern/Conduit/CMakeLists.txt b/Src/Extern/Conduit/CMakeLists.txt
@@ -4,6 +4,7 @@ foreach(D IN LISTS AMReX_SPACEDIM)
     # TODO: Particles PR merges another file
     target_sources(amrex_${D}d
        PRIVATE
+       AMReX_Conduit_Blueprint_ParticlesI.H
        AMReX_Conduit_Blueprint.H
        AMReX_Conduit_Blueprint.cpp
     )

diff --git a/Src/Extern/Conduit/Make.package b/Src/Extern/Conduit/Make.package
@@ -3,7 +3,7 @@
 #
 
 CEXE_sources += AMReX_Conduit_Blueprint.cpp
-
+CEXE_headers += AMReX_Conduit_Blueprint_ParticlesI.H
 CEXE_headers += AMReX_Conduit_Blueprint.H
 
 VPATH_LOCATIONS += $(AMREX_HOME)/Src/Extern/Conduit

diff --git a/Tools/CMake/AMReXConfig.cmake.in b/Tools/CMake/AMReXConfig.cmake.in
@@ -79,6 +79,7 @@ set(AMReX_PARTICLES_FOUND           @AMReX_PARTICLES@)
 set(AMReX_P@AMReX_PARTICLES_PRECISION@_FOUND ON)
 set(AMReX_SENSEI_FOUND              @AMReX_SENSEI@)
 set(AMReX_CONDUIT_FOUND             @AMReX_CONDUIT@)
+set(AMReX_CATALYST_FOUND            @AMReX_CATALYST@)
 set(AMReX_ASCENT_FOUND              @AMReX_ASCENT@)
 set(AMReX_HYPRE_FOUND               @AMReX_HYPRE@)
 set(AMReX_PETSC_FOUND               @AMReX_PETSC@)
@@ -134,6 +135,7 @@ set(AMReX_PARTICLES_PRECISION       @AMReX_PARTICLES_PRECISION@)
 set(AMReX_SENSEI                    @AMReX_SENSEI@)
 set(AMReX_NO_SENSEI_AMR_INST        @AMReX_NO_SENSEI_AMR_INST@)
 set(AMReX_CONDUIT                   @AMReX_CONDUIT@)
+set(AMReX_CATALYST                  @AMReX_CATALYST@)
 set(AMReX_ASCENT                    @AMReX_ASCENT@)
 set(AMReX_HYPRE                     @AMReX_HYPRE@)
 set(AMReX_PETSC                     @AMReX_PETSC@)
@@ -202,6 +204,10 @@ if (@AMReX_ASCENT@)
     find_dependency(Ascent REQUIRED)
 endif ()
 
+if (@AMReX_CATALYST@)
+    find_dependency(Catalyst REQUIRED)
+endif ()
+
 if (@AMReX_CONDUIT@)
    find_dependency(Conduit REQUIRED)
 endif ()

diff --git a/Tools/CMake/AMReXOptions.cmake b/Tools/CMake/AMReXOptions.cmake
@@ -320,6 +320,11 @@ print_option( AMReX_NO_SENSEI_AMR_INST )
 option( AMReX_CONDUIT "Enable Conduit support" OFF )
 print_option( AMReX_CONDUIT )
 
+# Catalyst
+cmake_dependent_option( AMReX_CATALYST "Enable Catalyst support" OFF 
+   "AMReX_CONDUIT" OFF )
+print_option( AMReX_CATALYST )
+
 # Ascent
 cmake_dependent_option( AMReX_ASCENT "Enable Ascent support" OFF
    "AMReX_CONDUIT" OFF )

diff --git a/Tools/CMake/AMReXSetDefines.cmake b/Tools/CMake/AMReXSetDefines.cmake
@@ -148,6 +148,9 @@ add_amrex_define( AMREX_NO_SENSEI_AMR_INST NO_LEGACY IF AMReX_NO_SENSEI_AMR_INST
 # Conduit Support
 add_amrex_define( AMREX_USE_CONDUIT NO_LEGACY IF AMReX_CONDUIT )
 
+# Catalyst Support
+add_amrex_define( AMREX_USE_CATALYST NO_LEGACY IF AMReX_CATALYST )
+
 # Ascent Support
 add_amrex_define( AMREX_USE_ASCENT NO_LEGACY IF AMReX_ASCENT )
 

diff --git a/Tools/CMake/AMReXThirdPartyLibraries.cmake b/Tools/CMake/AMReXThirdPartyLibraries.cmake
@@ -90,6 +90,15 @@ if (AMReX_ASCENT) # Ascent will find conduit, so check for Ascent first
     endforeach()
 endif ()
 
+#
+#  Catalyst
+#
+if (AMReX_CATALYST) 
+    find_package(Catalyst REQUIRED PATHS "$ENV{CATALYST_IMPLEMENTATION_PATHS}")
+    foreach(D IN LISTS AMReX_SPACEDIM)
+        target_link_libraries(amrex_${D}d PUBLIC catalyst::catalyst)
+    endforeach()
+endif ()
 
 #
 # Conduit

diff --git a/Tools/CMake/AMReX_Config_ND.H.in b/Tools/CMake/AMReX_Config_ND.H.in
@@ -42,6 +42,7 @@
 #cmakedefine AMREX_USE_SENSEI_INSITU
 #cmakedefine AMREX_NO_SENSEI_AMR_INST
 #cmakedefine AMREX_USE_CONDUIT
+#cmakedefine AMREX_USE_CATALYST
 #cmakedefine AMREX_USE_ASCENT
 #cmakedefine AMREX_USE_EB
 #cmakedefine AMREX_USE_CUDA

diff --git a/Tools/GNUMake/Make.defs b/Tools/GNUMake/Make.defs
@@ -216,6 +216,12 @@ else
   USE_CONDUIT := FALSE
 endif
 
+ifdef USE_CATALYST
+  USE_CATALYST := $(strip $(USE_CATALYST))
+else
+  USE_CATALYST := FALSE
+endif
+
 ifdef USE_ASCENT
   USE_ASCENT := $(strip $(USE_ASCENT))
 else
@@ -1057,6 +1063,11 @@ ifeq ($(USE_CONDUIT),TRUE)
   include        $(AMREX_HOME)/Tools/GNUMake/packages/Make.conduit
 endif
 
+ifeq ($(USE_CATALYST),TRUE)
+  $(info Loading $(AMREX_HOME)/Tools/GNUMake/packages/Make.catalyst...)
+  include        $(AMREX_HOME)/Tools/GNUMake/packages/Make.catalyst
+endif
+
 ifeq ($(USE_ASCENT),TRUE)
   $(info Loading $(AMREX_HOME)/Tools/GNUMake/packages/Make.ascent...)
   include        $(AMREX_HOME)/Tools/GNUMake/packages/Make.ascent

diff --git a/Tools/GNUMake/packages/Make.catalyst b/Tools/GNUMake/packages/Make.catalyst
@@ -0,0 +1,16 @@
+#########################################################
+# Catalyst (https://gitlab.kitware.com/paraview/catalyst) Support 
+#########################################################
+
+CPPFLAGS += -DAMREX_USE_CATALYST
+
+ifdef CATALYST_DIR
+  INCLUDE_LOCATIONS += $(CATALYST_DIR)/include
+  VPATH_LOCATIONS   += $(CATALYST_DIR)/include
+  LIBRARY_LOCATIONS += $(CATALYST_DIR)/lib
+  LIBRARIES += -Wl,-rpath,$(CATALYST_DIR)/lib
+
+  LIBRARIES += -lcatalyst
+
+endif
+