Merge pull request #290 from sethrj/trilinos-132

Update to support Trilinos 13.2

.jenkins

@@ -31,6 +31,7 @@ pipeline {
                         dir('build') {
                             sh '''
                                 cmake \
+                                  -GNinja \
                                   -D CMAKE_INSTALL_PREFIX=$FORTRILINOS_DIR \
                                   -D CMAKE_BUILD_TYPE=Debug \
                                   -D CMAKE_CXX_COMPILER_LAUNCHER=ccache \
@@ -39,7 +40,7 @@ pipeline {
                                   -D ForTrilinos_EXAMPLES=ON \
                                 ..
                             '''
-                            sh 'make -j8 VERBOSE=1'
+                            sh 'ninja -v'
                             sh 'ctest $CTEST_OPTIONS'
                         }
                     }
@@ -49,7 +50,7 @@ pipeline {
                             xunit reduceLog: false, tools:[CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)]
                         }
                         success {
-                            sh 'cd build && make install'
+                            sh 'cd build && ninja install'
                             sh 'rm -rf test_install && mkdir -p test_install'
                             dir('test_install') {
                                 sh 'cp -r ../example/test-installation .'

doc/install.rst

@@ -27,8 +27,8 @@ The version scheme is based on semantic versioning:
   changes for ForTrilinos.
 
 - New features in Trilinos, and new support by ForTrilinos, can result in minor
-  version number changes. Features removed by a minor version change in
-  Trilinos may also result in a minor version change.
+  version number changes. Features removed or deprecated by a minor version
+  change in Trilinos may also result in a minor version change.
 
 - Minor changes to the SWIG-Fortran implementation (which don't affect the
   interface in the .F90 files) result in a patch version.
@@ -43,6 +43,7 @@ in the committed version of the generated wrappers.
    ===========  ============== ======================
    ForTrilinos  Trilinos       SWIG
    ===========  ============== ======================
+   2.1.0        13.2           4.1.0-dev1+fortran
    2.0.1        13.0:13.1      4.1.0-dev1+fortran
    2.0.0        13.0:13.1      4.0.2+fortran
    2.0.0-dev3   12.18.1        4.0.2+fortran
@@ -72,8 +73,8 @@ pre-built binary on a variety of user and HPC systems.
 Spack
 -----
 
-To install ForTrilinos version ``2.0.1`` through an existing Spack
-installation (v0.18 or higher, or the ``develop`` branch):
+To install ForTrilinos version ``2.1.0`` through an existing Spack
+installation (v0.19 or higher, or the ``develop`` branch):
 
 .. code:: console
 

docker/Dockerfile

@@ -1,94 +1,68 @@
 # Build command:
-#   $ docker build -t aprokop/fortrilinos-stack:latest -f Dockerfile  .
-ARG BASE=nvidia/cuda:11.0.3-devel-ubuntu18.04
+#   $ docker build -t sethrj/fortrilinos-stack:latest -f Dockerfile  .
+ARG BASE=nvidia/cuda:11.2.2-devel-ubuntu20.04
 FROM $BASE
 
 ARG NPROC=8
 
-RUN if test ${NV_CUDA_LIB_VERSION}; then apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub; fi
-
 # Avoid tzdata dialog
 ARG DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -yq \
+RUN apt-get -yqq update && apt-get install -yq \
         autoconf \
         bc \
         build-essential \
+        ca-certificates \
         ccache \
         curl \
-        environment-modules \
         gawk \
         gfortran \
+        g++ \
         git \
         lcov \
         libatlas-base-dev \
         libbz2-dev \
-        python2.7-dev \
-        tmux \
-        unzip \
+        libssl-dev \
+        mpich \
+        ninja-build \
+        python3-dev \
         valgrind \
         vim \
         wget \
         && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Install CMake
 ENV CMAKE_DIR=/opt/cmake
-RUN CMAKE_VERSION=3.16.4 && \
-    CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION} && \
-    CMAKE_SCRIPT=cmake-${CMAKE_VERSION}-Linux-x86_64.sh && \
-    CMAKE_SHA256=cmake-${CMAKE_VERSION}-SHA-256.txt && \
+RUN CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v3.23.3/cmake-3.23.3.tar.gz && \
+    CMAKE_ARCHIVE=cmake.tar.gz && \
     SCRATCH_DIR=/scratch && mkdir -p ${SCRATCH_DIR} && cd ${SCRATCH_DIR} && \
-    wget --quiet ${CMAKE_URL}/${CMAKE_SHA256} && \
-    wget --quiet ${CMAKE_URL}/${CMAKE_SHA256}.asc && \
-    wget --quiet ${CMAKE_URL}/${CMAKE_SCRIPT} && \
-    grep ${CMAKE_SCRIPT} ${CMAKE_SHA256} | sha256sum --check && \
+    wget --quiet ${CMAKE_URL}  --output-document=${CMAKE_ARCHIVE} && \
     mkdir -p ${CMAKE_DIR} && \
-    sh ${CMAKE_SCRIPT} --skip-license --prefix=${CMAKE_DIR} && \
+    tar -xf ${CMAKE_ARCHIVE} --strip-components=1 && \
+    ./bootstrap --prefix=${CMAKE_DIR} --generator=Ninja && \
+    ninja install && \
     rm -rf ${SCRATCH_DIR}
 ENV PATH=${CMAKE_DIR}/bin:$PATH
 
-# Install OpenMPI
-ENV OPENMPI_DIR=/opt/openmpi
-RUN OPENMPI_VERSION=4.0.3 && \
-    OPENMPI_VERSION_SHORT=$(echo "$OPENMPI_VERSION" | cut -d. -f1,2) && \
-    OPENMPI_SHA1=d958454e32da2c86dd32b7d557cf9a401f0a08d3 && \
-    OPENMPI_URL=https://download.open-mpi.org/release/open-mpi/v${OPENMPI_VERSION_SHORT}/openmpi-${OPENMPI_VERSION}.tar.bz2 && \
-    OPENMPI_ARCHIVE=openmpi-${OPENMPI_VERSION}.tar.bz2 && \
-    SCRATCH_DIR=/scratch && mkdir -p ${SCRATCH_DIR} && cd ${SCRATCH_DIR} && \
-    wget --quiet ${OPENMPI_URL} --output-document=${OPENMPI_ARCHIVE} && \
-    echo "${OPENMPI_SHA1} ${OPENMPI_ARCHIVE}" | sha1sum -c && \
-    mkdir -p openmpi && \
-    tar -xf ${OPENMPI_ARCHIVE} -C openmpi --strip-components=1 && \
-    mkdir -p build && cd build && \
-    ../openmpi/configure --prefix=${OPENMPI_DIR} ${CUDA_OPTIONS} CFLAGS=-w && \
-    make -j${NPROCS} install && \
-    rm -rf ${SCRATCH_DIR}
-ENV PATH=${OPENMPI_DIR}/bin:$PATH
-
-# Workaround for Kokkos to find libcudart
-ENV LD_LIBRARY_PATH=/usr/local/cuda/targets/x86_64-linux/lib:${LD_LIBRARY_PATH}
-
-# Install Trilinos (13.0.0)
+# Install Trilinos
 ENV TRILINOS_DIR=/opt/trilinos
-RUN export TRILINOS_HASH=9fec35276d846a667bc668ff4cbdfd8be0dfea08 && \
-    export TRILINOS_URL=https://github.com/trilinos/Trilinos/archive/${TRILINOS_HASH}.tar.gz && \
-    export TRILINOS_ARCHIVE=trilinos-${TRILINOS_HASH}.tar.gz && \
+RUN TRILINOS_VERSION=13-2-0 && \
+    TRILINOS_URL=https://github.com/trilinos/Trilinos/archive/refs/tags/trilinos-release-${TRILINOS_VERSION}.tar.gz && \
+    TRILINOS_ARCHIVE=trilinos-${TRILINOS_VERSION}.tar.gz && \
     SCRATCH_DIR=/scratch && mkdir -p ${SCRATCH_DIR} && cd ${SCRATCH_DIR} && \
     wget --quiet ${TRILINOS_URL} --output-document=${TRILINOS_ARCHIVE} && \
     mkdir -p trilinos && \
     tar -xf ${TRILINOS_ARCHIVE} -C trilinos --strip-components=1 && \
     mkdir -p build && cd build && \
     cmake \
+        -G Ninja \
         -D CMAKE_BUILD_TYPE=RelWithDebInfo \
         -D BUILD_SHARED_LIBS=ON \
         \
         -D CMAKE_INSTALL_PREFIX=${TRILINOS_DIR} \
         \
         -D TPL_ENABLE_Boost=OFF \
-              -D TPL_ENABLE_CUDA=ON \
-        -D TPL_ENABLE_CUDA=OFF \
         -D TPL_ENABLE_DLlib=OFF \
         -D TPL_ENABLE_MPI=ON \
         \
@@ -129,6 +103,6 @@ RUN export TRILINOS_HASH=9fec35276d846a667bc668ff4cbdfd8be0dfea08 && \
             -D Tpetra_ENABLE_DEPRECATED_CODE=OFF \
         \
     ../trilinos && \
-    make -j${NPROC} install && \
+    ninja -j${NPROC} install && \
     rm -rf ${SCRATCH_DIR}
 ENV TRILINOS_DIR=/opt/trilinos

scripts/spack.yaml

@@ -4,7 +4,7 @@ spack:
   - ninja
   - openmpi
   - swig@4.1.0-fortran
-  - trilinos@13.0.0:13.1 +amesos2 +anasazi +belos +kokkos +ifpack2 +muelu +nox +tpetra
+  - trilinos@13.2.0:13.2 +amesos2 +anasazi +belos +kokkos +ifpack2 +muelu +nox +tpetra
     +stratimikos +mpi gotype=long_long
   view: true
   concretizer:

src/fortpetra/Tpetra_CrsGraph.i

@@ -111,6 +111,20 @@
       for (int i = 0; i < nnz; i++)
         columnIndices[i] = columnIndicesArrayRCP[i]+1;
     }
+    void
+    getLocalRowCopy (LO lclRow,
+                     const Teuchos::ArrayView<LO>& lclColInds,
+                     size_t& numColInds) const {
+        Tpetra::CrsGraph<LO,GO,NO>::nonconst_local_inds_host_view_type temp_i(lclColInds.data(), lclColInds.size());
+        return $self->getLocalRowCopy(lclRow, temp_i, numColInds);
+    }
+    void
+    getGlobalRowCopy (GO gblRow,
+                     const Teuchos::ArrayView<GO>& gblColInds,
+                     size_t& numColInds) const {
+        Tpetra::CrsGraph<LO,GO,NO>::nonconst_global_inds_host_view_type temp_i(gblColInds.data(), gblColInds.size());
+        return $self->getGlobalRowCopy(gblRow, temp_i, numColInds);
+    }
 }
 
 // Add doImport and doExport

src/fortpetra/Tpetra_CrsMatrix.i

@@ -68,7 +68,7 @@
     const Teuchos::ArrayView<const LO>& cols,
     Teuchos::ArrayView<const LO> cols}
 
-%apply const Teuchos::ArrayView<int>& INDEX { const Teuchos::ArrayView<LO>& colInds }
+%apply const Teuchos::ArrayView<int>& INDEX { const Teuchos::ArrayView<LO>& colInds, const Teuchos::ArrayView<LO>& colInds }
 
 %apply const Teuchos::ArrayRCP<const int>& INDEX {
     const Teuchos::ArrayRCP<size_t>& rowPointers,
@@ -104,6 +104,26 @@
         values       [i] = valuesArrayRCP[i];
       }
     }
+
+    void
+    getGlobalRowCopy (GO GlobalRow,
+                      const Teuchos::ArrayView<GO>& Indices,
+                      const Teuchos::ArrayView<SC>& Values,
+                      size_t& NumEntries) const {
+        Tpetra::CrsMatrix<SC,LO,GO,NO>::nonconst_global_inds_host_view_type temp_i(Indices.data(), Indices.size());
+        Tpetra::CrsMatrix<SC,LO,GO,NO>::nonconst_values_host_view_type temp_v(Values.data(), Values.size());
+        return $self->getGlobalRowCopy(GlobalRow, temp_i, temp_v, NumEntries);
+    }
+
+    void
+    getLocalRowCopy (LO localRow,
+                     const Teuchos::ArrayView<LO>& colInds,
+                     const Teuchos::ArrayView<SC>& vals,
+                     size_t& numEntries) const {
+        Tpetra::CrsMatrix<SC,LO,GO,NO>::nonconst_local_inds_host_view_type temp_i(colInds.data(), colInds.size());
+        Tpetra::CrsMatrix<SC,LO,GO,NO>::nonconst_values_host_view_type temp_v(vals.data(), vals.size());
+        return $self->getLocalRowCopy(localRow, temp_i, temp_v, numEntries);
+    }
 }
 
 // Add doImport and doExport
@@ -116,9 +136,59 @@
 %ignore Tpetra::CrsMatrix::sumIntoLocalValues (const LocalOrdinal localRow, const LocalOrdinal numEnt, const Scalar vals[], const LocalOrdinal cols[], const bool atomic=useAtomicUpdatesByDefault);
 
 
+// Define type aliases for Tpetra matrix base class
+namespace Tpetra {
+  template <class Scalar,
+            class LocalOrdinal,
+            class GlobalOrdinal,
+            class Node>
+  class RowMatrix
+{
+public:
+    typedef Scalar        scalar_type;
+    typedef LocalOrdinal  local_ordinal_type;
+    typedef GlobalOrdinal global_ordinal_type;
+    typedef Node          node_type;
+
+    using impl_scalar_type = typename Kokkos::ArithTraits<Scalar>::val_type;
+    using mag_type = typename Kokkos::ArithTraits<Scalar>::mag_type;
+    typedef typename
+        Kokkos::View<impl_scalar_type*, typename Node::device_type>::const_type
+        values_device_view_type;
+    typedef typename values_device_view_type::HostMirror::const_type
+        values_host_view_type;
+    typedef typename values_device_view_type::HostMirror
+        nonconst_values_host_view_type;
+
+    typedef typename
+        Kokkos::View<LocalOrdinal *, typename Node::device_type>::const_type
+        local_inds_device_view_type;
+    typedef typename local_inds_device_view_type::HostMirror::const_type
+        local_inds_host_view_type;
+    typedef typename local_inds_device_view_type::HostMirror
+        nonconst_local_inds_host_view_type;
+
+    typedef typename
+        Kokkos::View<GlobalOrdinal *, typename Node::device_type>::const_type
+        global_inds_device_view_type;
+    typedef typename global_inds_device_view_type::HostMirror::const_type
+        global_inds_host_view_type;
+    typedef typename global_inds_device_view_type::HostMirror
+        nonconst_global_inds_host_view_type;
+
+
+    typedef typename
+        Kokkos::View<const size_t*, typename Node::device_type>::const_type
+        row_ptrs_device_view_type;
+    typedef typename row_ptrs_device_view_type::HostMirror::const_type
+        row_ptrs_host_view_type;
+};
+}
+
 %include "Tpetra_CrsMatrix_decl.hpp"
 
 %teuchos_rcp(Tpetra::CrsMatrix<SC,LO,GO,NO>)
+%template() Tpetra::RowMatrix<SC,LO,GO,NO>;
 %template(TpetraCrsMatrix) Tpetra::CrsMatrix<SC,LO,GO,NO>;
 
 // Operator to Matrix conversion