Tpetra LAI

.travis.yml

@@ -2,7 +2,7 @@ language: minimal
 
 env:
   global:
-  - GEOSX_TPL_TAG=123-507
+  - GEOSX_TPL_TAG=120-511
   - secure: CGs2uH6efq1Me6xJWRr0BnwtwxoujzlowC4FHXHdWbNOkPsXf7nCgdaW5vthfD3bhnOeEUQSrfxdhTRtyU/NfcKLmKgGBnZOdUG4/JJK4gDSJ2Wp8LZ/mB0QEoODKVxbh+YtoAiHe3y4M9PGCs+wkNDw/3eEU00cK12DZ6gad0RbLjI3xkhEr/ZEZDZkcYg9yHAhl5bmpqoh/6QGnIg8mxIqdAtGDw+6tT0EgUqjeqc5bG5WwsamKzJItHSXD5zx8IJAlgDk4EzEGjZe0m56YnNfb9iwqqUsmL3Cuwgs7ByVDYw78JC5Kv42YqoxA5BxMT2mFsEe37TpYNXlzofU7ma2Duw9DGXWQd4IkTCcBxlyR0I0bfo0TmgO+y7PYG9lIyHPUkENemdozsZcWamqqkqegiEdRhDVYlSRo3mu7iCwTS6ZTALliVyEYjYxYb7oAnR3cNywXjblTCI8oKfgLSY+8WijM9SRl57JruIHLkLMCjmRI+cZBfv5tS2tYQTBPkygGrigrrN77ZiC7/TGyfggSN0+y0oYtOAgqEnBcKcreiibMW7tKcV2Z1RFD9ZvIkSc1EXLUPDP8FX1oyhmqBMqVo8LksrYLDJHQ05+F3YNgl2taSt7uMjQ4e8iZ3/IjFeMnbylDw+cj/RbS520HXsFPbWFm2Pb9pceA9n6GnY=
 
 # The integrated test repository contains large data (using git lfs) and we do not use them here.
@@ -17,7 +17,7 @@ geosx_before_script: &geosx_before_script
   - git submodule update --init src/externalComponents/PAMELA
   - git submodule update --init --recursive src/coreComponents/fileIO/coupling/hdf5_interface
   - git submodule update --init --recursive src/coreComponents/physicsSolvers/GEOSX_PTP
-  
+
 
 geosx_linux_build: &geosx_linux_build
   services: docker

src/cmake/GeosxOptions.cmake

@@ -51,20 +51,22 @@ option( ENABLE_SUITESPARSE "Enables SUITESPARSE" ON )
 
 ### LAI SETUP ###
 
-set( supported_LAI Trilinos Hypre Petsc )
+set( supported_LAI Trilinos TrilinosTpetra Hypre Petsc )
 set( GEOSX_LA_INTERFACE "Trilinos" CACHE STRING "Linear algebra interface to use in solvers" )
-message( STATUS "GEOSX_LA_INTERFACE = ${GEOSX_LA_INTERFACE}" )
+message( "GEOSX_LA_INTERFACE = ${GEOSX_LA_INTERFACE}" )
 
 if( NOT ( GEOSX_LA_INTERFACE IN_LIST supported_LAI ) )
   message( FATAL_ERROR "GEOSX_LA_INTERFACE must be one of: ${supported_LAI}" )
 endif()
 
 string( TOUPPER "${GEOSX_LA_INTERFACE}" upper_LAI )
-if( NOT ENABLE_${upper_LAI} )
-  message( FATAL_ERROR "${GEOSX_LA_INTERFACE} LA interface is selected, but ENABLE_${upper_LAI} is OFF" )
-endif()
 option( GEOSX_LA_INTERFACE_${upper_LAI} "${upper_LAI} LA interface is selected" ON )
 
+string( REPLACE "TPETRA" "" dep_LAI "${upper_LAI}" )
+if( NOT ENABLE_${dep_LAI} )
+  message( FATAL_ERROR "${GEOSX_LA_INTERFACE} LA interface is selected, but ENABLE_${dep_LAI} is OFF" )
+endif()
+
 ### MPI/OMP/CUDA SETUP ###
 
 option( ENABLE_MPI "" ON )

src/coreComponents/common/GeosxConfig.hpp.in

@@ -75,6 +75,8 @@
 #cmakedefine GEOSX_LA_INTERFACE @GEOSX_LA_INTERFACE@
 /// Macro defined when Trilinos interface is selected
 #cmakedefine GEOSX_LA_INTERFACE_TRILINOS
+/// Macro defined when Trilinos interface is selected
+#cmakedefine GEOSX_LA_INTERFACE_TRILINOSTPETRA
 /// Macro defined when Hypre interface is selected
 #cmakedefine GEOSX_LA_INTERFACE_HYPRE
 /// Macro defined when PETSc interface is selected

src/coreComponents/fileIO/schema/schema.xsd

@@ -575,14 +575,38 @@
 		<xsd:attribute name="name" type="string" use="required" />
 	</xsd:complexType>
 	<xsd:complexType name="LinearSolverParametersType">
-		<!--amgCoarseSolver => AMG coarsest level solver/smoother type
-Available options are: jacobi, gaussSeidel, blockGaussSeidel, chebyshev, direct-->
-		<xsd:attribute name="amgCoarseSolver" type="string" default="direct" />
+		<!--amgCoarseSolver => AMG coarsest level solver/smoother type. Valid options (not all may be supported by linear algebra package):
+* none
+* jacobi
+* gs
+* sgs
+* chebyshev
+* iluk
+* ilut
+* icc
+* ict
+* amg
+* mgr
+* block
+* direct-->
+		<xsd:attribute name="amgCoarseSolver" type="geosx_LinearSolverParameters_PreconditionerType" default="direct" />
 		<!--amgNumSweeps => AMG smoother sweeps-->
 		<xsd:attribute name="amgNumSweeps" type="integer" default="2" />
-		<!--amgSmootherType => AMG smoother type
-Available options are: jacobi, blockJacobi, gaussSeidel, blockGaussSeidel, chebyshev, icc, ilu, ilut-->
-		<xsd:attribute name="amgSmootherType" type="string" default="gaussSeidel" />
+		<!--amgSmootherType => AMG smoother type. Valid options (not all may be supported by linear algebra package):
+* none
+* jacobi
+* gs
+* sgs
+* chebyshev
+* iluk
+* ilut
+* icc
+* ict
+* amg
+* mgr
+* block
+* direct-->
+		<xsd:attribute name="amgSmootherType" type="geosx_LinearSolverParameters_PreconditionerType" default="gs" />
 		<!--amgThreshold => AMG strength-of-connection threshold-->
 		<xsd:attribute name="amgThreshold" type="real64" default="0" />
 		<!--directCheckResTol => Tolerance used to check a direct solver solution-->
@@ -631,13 +655,15 @@ the relative residual norm satisfies:
 * jacobi
 * gs
 * sgs
+* chebyshev
 * iluk
 * ilut
 * icc
 * ict
 * amg
 * mgr
-* block-->
+* block
+* direct-->
 		<xsd:attribute name="preconditionerType" type="geosx_LinearSolverParameters_PreconditionerType" default="iluk" />
 		<!--solverType => Linear solver type. Available options are:
 * direct
@@ -650,6 +676,11 @@ the relative residual norm satisfies:
 		<!--stopIfError => Whether to stop the simulation if the linear solver reports an error-->
 		<xsd:attribute name="stopIfError" type="integer" default="1" />
 	</xsd:complexType>
+	<xsd:simpleType name="geosx_LinearSolverParameters_PreconditionerType">
+		<xsd:restriction base="xsd:string">
+			<xsd:pattern value=".*[\[\]`$].*|none|jacobi|gs|sgs|chebyshev|iluk|ilut|icc|ict|amg|mgr|block|direct" />
+		</xsd:restriction>
+	</xsd:simpleType>
 	<xsd:simpleType name="geosx_LinearSolverParameters_Direct_ColPerm">
 		<xsd:restriction base="xsd:string">
 			<xsd:pattern value=".*[\[\]`$].*|none|MMD_AtplusA|MMD_AtA|colAMD|metis|parmetis" />
@@ -660,11 +691,6 @@ the relative residual norm satisfies:
 			<xsd:pattern value=".*[\[\]`$].*|none|mc64" />
 		</xsd:restriction>
 	</xsd:simpleType>
-	<xsd:simpleType name="geosx_LinearSolverParameters_PreconditionerType">
-		<xsd:restriction base="xsd:string">
-			<xsd:pattern value=".*[\[\]`$].*|none|jacobi|gs|sgs|iluk|ilut|icc|ict|amg|mgr|block" />
-		</xsd:restriction>
-	</xsd:simpleType>
 	<xsd:simpleType name="geosx_LinearSolverParameters_SolverType">
 		<xsd:restriction base="xsd:string">
 			<xsd:pattern value=".*[\[\]`$].*|direct|cg|gmres|fgmres|bicgstab|preconditioner" />

src/coreComponents/finiteVolume/CMakeLists.txt

@@ -26,9 +26,9 @@ set( finiteVolume_sources
    )
 
 if ( BUILD_OBJ_LIBS )
-  set( dependencyList dataRepository codingUtilities managers)
+  set( dependencyList dataRepository codingUtilities managers )
 else()
-  set( dependencyList common)
+  set( dependencyList common managers )
 endif()
 
 if ( ENABLE_OPENMP )

src/coreComponents/linearAlgebra/CMakeLists.txt

@@ -63,16 +63,27 @@ if( ENABLE_TRILINOS )
           interfaces/trilinos/EpetraMatrix.hpp
           interfaces/trilinos/EpetraVector.hpp
           interfaces/trilinos/EpetraUtils.hpp
+          interfaces/trilinos/TpetraMatrix.hpp
+          interfaces/trilinos/TpetraVector.hpp
+          interfaces/trilinos/TpetraUtils.hpp
           interfaces/trilinos/TrilinosPreconditioner.hpp
+          interfaces/trilinos/TrilinosTpetraPreconditioner.hpp
           interfaces/trilinos/TrilinosSolver.hpp
-          interfaces/trilinos/TrilinosInterface.hpp )
+          interfaces/trilinos/TrilinosTpetraSolver.hpp
+          interfaces/trilinos/TrilinosInterface.hpp
+          interfaces/trilinos/TrilinosTpetraInterface.hpp )
 
     list( APPEND linearAlgebra_sources
           interfaces/trilinos/EpetraMatrix.cpp
           interfaces/trilinos/EpetraVector.cpp
+          interfaces/trilinos/TpetraMatrix.cpp
+          interfaces/trilinos/TpetraVector.cpp
           interfaces/trilinos/TrilinosPreconditioner.cpp
+          interfaces/trilinos/TrilinosTpetraPreconditioner.cpp
           interfaces/trilinos/TrilinosSolver.cpp
-          interfaces/trilinos/TrilinosInterface.cpp )
+          interfaces/trilinos/TrilinosTpetraSolver.cpp
+          interfaces/trilinos/TrilinosInterface.cpp
+          interfaces/trilinos/TrilinosTpetraInterface.cpp )
 
     list( APPEND dependencyList trilinos )
 

src/coreComponents/linearAlgebra/DofManager.cpp

@@ -1983,6 +1983,7 @@ void DofManager::printFieldInfo( std::ostream & os ) const
 
 #ifdef GEOSX_USE_TRILINOS
 MAKE_DOFMANAGER_METHOD_INST( TrilinosInterface )
+MAKE_DOFMANAGER_METHOD_INST( TrilinosTpetraInterface )
 #endif
 
 #ifdef GEOSX_USE_HYPRE

src/coreComponents/linearAlgebra/interfaces/InterfaceTypes.hpp

@@ -23,6 +23,7 @@
 
 #ifdef GEOSX_USE_TRILINOS
 #include "linearAlgebra/interfaces/trilinos/TrilinosInterface.hpp"
+#include "linearAlgebra/interfaces/trilinos/TrilinosTpetraInterface.hpp"
 #endif
 
 #ifdef GEOSX_USE_HYPRE
@@ -62,6 +63,7 @@ inline void setupLAI( int & argc, char * * & argv )
 {
 #ifdef GEOSX_USE_TRILINOS
   TrilinosInterface::initialize( argc, argv );
+  TrilinosTpetraInterface::initialize( argc, argv );
 #endif
 #ifdef GEOSX_USE_HYPRE
   HypreInterface::initialize( argc, argv );
@@ -78,6 +80,7 @@ inline void finalizeLAI()
 {
 #ifdef GEOSX_USE_TRILINOS
   TrilinosInterface::finalize();
+  TrilinosTpetraInterface::finalize();
 #endif
 #ifdef GEOSX_USE_HYPRE
   HypreInterface::finalize();

src/coreComponents/linearAlgebra/interfaces/MatrixBase.hpp

@@ -21,7 +21,6 @@
 
 #include "linearAlgebra/common.hpp"
 #include "linearAlgebra/interfaces/LinearOperator.hpp"
-//#include "LvArray/src/streamIO.hpp"
 
 namespace geosx
 {
@@ -438,36 +437,6 @@ class MatrixBase : public virtual LinearOperator< VECTOR >
                        arraySlice1d< globalIndex const > const & colIndices,
                        arraySlice2d< real64 const, MatrixLayout::ROW_MAJOR > const & values ) = 0;
 
-  /**
-   * @brief Add a dense block of values.
-   * @param rowIndices Global row indices
-   * @param colIndices Global col indices
-   * @param values Dense local matrix of values
-   */
-  virtual void add( arraySlice1d< globalIndex const > const & rowIndices,
-                    arraySlice1d< globalIndex const > const & colIndices,
-                    arraySlice2d< real64 const, MatrixLayout::COL_MAJOR > const & values ) = 0;
-
-  /**
-   * @brief Set a dense block of values.
-   * @param rowIndices Global row indices
-   * @param colIndices Global col indices
-   * @param values Dense local matrix of values
-   */
-  virtual void set( arraySlice1d< globalIndex const > const & rowIndices,
-                    arraySlice1d< globalIndex const > const & colIndices,
-                    arraySlice2d< real64 const, MatrixLayout::COL_MAJOR > const & values ) = 0;
-
-  /**
-   * @brief Insert a dense block of values.
-   * @param rowIndices Global row indices
-   * @param colIndices Global col indices
-   * @param values Dense local matrix of values
-   */
-  virtual void insert( arraySlice1d< globalIndex const > const & rowIndices,
-                       arraySlice1d< globalIndex const > const & colIndices,
-                       arraySlice2d< real64 const, MatrixLayout::COL_MAJOR > const & values ) = 0;
-
   /**
    * @brief Add a dense block of values.
    * @param rowIndices Global row indices

src/coreComponents/linearAlgebra/interfaces/VectorBase.hpp

@@ -149,7 +149,7 @@ class VectorBase
    * @param localValues local data to put into vector
    * @param comm MPI communicator to use
    */
-  virtual void create( arrayView1d< real64 const > const & localValues, MPI_Comm const & comm ) = 0;
+  virtual void createWithLocalValues( arrayView1d< real64 > const & localValues, MPI_Comm const & comm ) = 0;
 
   ///@}
 
@@ -414,6 +414,7 @@ class VectorBase
    */
   virtual void extract( arrayView1d< real64 > const & localVector ) const
   {
+    GEOSX_LAI_ASSERT_EQ( localSize(), localVector.size() );
     real64 const * const data = extractLocalVector();
     forAll< parallelHostPolicy >( localSize(), [=] ( localIndex const k )
     {

src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.cpp

@@ -25,6 +25,7 @@
 #include "HypreUtils.hpp"
 
 #include <iomanip>
+#include <numeric>
 
 namespace geosx
 {
@@ -447,53 +448,6 @@ void HypreMatrix::insert( arraySlice1d< globalIndex const > const & rowIndices,
   }
 }
 
-namespace
-{
-
-template< typename T, int SRC_USD, int DST_USD >
-static void convertArrayLayout( arraySlice2d< T const, SRC_USD > const & src,
-                                arraySlice2d< T, DST_USD > const & dst )
-{
-  GEOSX_ASSERT( src.size( 0 ) == dst.size( 0 ) );
-  GEOSX_ASSERT( src.size( 1 ) == dst.size( 1 ) );
-  for( localIndex i = 0; i < src.size( 0 ); ++i )
-  {
-    for( localIndex j = 0; j < src.size( 1 ); ++j )
-    {
-      dst( i, j ) = src( i, j );
-    }
-  }
-}
-
-}
-
-void HypreMatrix::add( arraySlice1d< globalIndex const > const & rowIndices,
-                       arraySlice1d< globalIndex const > const & colIndices,
-                       arraySlice2d< real64 const, MatrixLayout::COL_MAJOR > const & values )
-{
-  array2d< real64, MatrixLayout::ROW_MAJOR_PERM > valuesRowMajor( rowIndices.size(), colIndices.size() );
-  convertArrayLayout( values, valuesRowMajor.toSlice() );
-  add( rowIndices, colIndices, valuesRowMajor );
-}
-
-void HypreMatrix::set( arraySlice1d< globalIndex const > const & rowIndices,
-                       arraySlice1d< globalIndex const > const & colIndices,
-                       arraySlice2d< real64 const, MatrixLayout::COL_MAJOR > const & values )
-{
-  array2d< real64, MatrixLayout::ROW_MAJOR_PERM > valuesRowMajor( rowIndices.size(), colIndices.size() );
-  convertArrayLayout( values, valuesRowMajor.toSlice() );
-  set( rowIndices, colIndices, valuesRowMajor );
-}
-
-void HypreMatrix::insert( arraySlice1d< globalIndex const > const & rowIndices,
-                          arraySlice1d< globalIndex const > const & colIndices,
-                          arraySlice2d< real64 const, MatrixLayout::COL_MAJOR > const & values )
-{
-  array2d< real64, MatrixLayout::ROW_MAJOR_PERM > valuesRowMajor( rowIndices.size(), colIndices.size() );
-  convertArrayLayout( values, valuesRowMajor.toSlice() );
-  insert( rowIndices, colIndices, valuesRowMajor );
-}
-
 void HypreMatrix::add( globalIndex const * rowIndices,
                        globalIndex const * colIndices,
                        real64 const * values,
@@ -908,10 +862,7 @@ localIndex HypreMatrix::maxRowLength() const
   array1d< HYPRE_BigInt > rows( nrows );
   array1d< HYPRE_Int > ncols( nrows );
 
-  for( HYPRE_Int i = 0; i < nrows; ++i )
-  {
-    rows[i] = ilower() + LvArray::integerConversion< HYPRE_BigInt >( i );
-  }
+  std::iota( rows.begin(), rows.end(), ilower() );
 
   GEOSX_LAI_CHECK_ERROR( HYPRE_IJMatrixGetRowCounts( m_ij_mat,
                                                      nrows,

src/coreComponents/linearAlgebra/interfaces/hypre/HypreMatrix.hpp

@@ -177,18 +177,6 @@ class HypreMatrix final : public virtual LinearOperator< HypreVector >,
                        arraySlice1d< globalIndex const > const & colIndices,
                        arraySlice2d< real64 const, MatrixLayout::ROW_MAJOR > const & values ) override;
 
-  virtual void add( arraySlice1d< globalIndex const > const & rowIndices,
-                    arraySlice1d< globalIndex const > const & colIndices,
-                    arraySlice2d< real64 const, MatrixLayout::COL_MAJOR > const & values ) override;
-
-  virtual void set( arraySlice1d< globalIndex const > const & rowIndices,
-                    arraySlice1d< globalIndex const > const & colIndices,
-                    arraySlice2d< real64 const, MatrixLayout::COL_MAJOR > const & values ) override;
-
-  virtual void insert( arraySlice1d< globalIndex const > const & rowIndices,
-                       arraySlice1d< globalIndex const > const & colIndices,
-                       arraySlice2d< real64 const, MatrixLayout::COL_MAJOR > const & values ) override;
-
   virtual void add( globalIndex const * rowIndices,
                     globalIndex const * colIndices,
                     real64 const * values,