Merge pull request #168 from LLNL/release/v2.3.0

Release v2.3.0

CMakeLists.txt

@@ -7,7 +7,7 @@
 cmake_policy(SET CMP0057 NEW)
 cmake_policy(SET CMP0048 NEW)
 
-project(Chai LANGUAGES CXX VERSION 2.2.2)
+project(Chai LANGUAGES CXX VERSION 2.3.0)
 
 set(ENABLE_CUDA Off CACHE BOOL "Enable CUDA")
 set(ENABLE_HIP Off CACHE BOOL "Enable HIP")

README.md

@@ -1,4 +1,4 @@
-# CHAI v2.2
+# CHAI v2.3
 
 [![Azure Build Status](https://dev.azure.com/davidbeckingsale/CHAI/_apis/build/status/LLNL.CHAI?branchName=develop)](https://dev.azure.com/davidbeckingsale/CHAI/_build/latest?definitionId=2&branchName=develop)
 [![Build Status](https://travis-ci.org/LLNL/CHAI.svg?branch=develop)](https://travis-ci.org/LLNL/CHAI)

docs/sphinx/conf.py

@@ -61,9 +61,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'2.2'
+version = u'2.3'
 # The full version, including alpha/beta/rc tags.
-release = u'2.2.2'
+release = u'2.3.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/sphinx/conf.py.in

@@ -60,9 +60,9 @@ author = u''
 # built documents.
 #
 # The short X.Y version.
-version = u'2.2'
+version = u'2.3'
 # The full version, including alpha/beta/rc tags.
-release = u'2.2.2'
+release = u'2.3.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/sphinx/developer/ci.rst

@@ -0,0 +1,14 @@
+.. _ci:
+
+======================
+Continuous Integration
+======================
+
+Gitlab CI
+---------
+
+CHAI shares its Gitlab CI workflow with other projects. The documentation is
+therefore `shared`_.
+
+.. shared: <https://radiuss-ci.readthedocs.io/en/latest/uberenv.html#ci)
+

docs/sphinx/developer/uberenv.rst

@@ -0,0 +1,106 @@
+.. _developer_guide:
+
+===============
+Developer Guide
+===============
+
+CHAI shares its Uberenv workflow with other projects. The documentation is
+therefore `shared`_.
+
+.. shared: <https://radiuss-ci.readthedocs.io/en/latest/uberenv.html#uberenv-guide)
+
+This page will provides some CHAI specific examples to illustrate the
+workflow described in the documentation.
+
+Machine specific configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: bash
+
+  $ ls -c1 scripts/uberenv/spack_configs
+  blueos_3_ppc64le_ib
+  darwin
+  toss_3_x86_64_ib
+  blueos_3_ppc64le_ib_p9
+  config.yaml
+
+CHAI has been configured for ``toss_3_x86_64_ib`` and other systems.
+
+Vetted specs
+^^^^^^^^^^^^
+
+.. code-block:: bash
+
+  $ ls -c1 .gitlab/*jobs.yml
+  .gitlab/lassen-jobs.yml
+  .gitlab/quartz-jobs.yml
+
+CI contains jobs for quartz.
+
+.. code-block:: bash
+
+  $ git grep -h "SPEC" .gitlab/quartz-jobs.yml | grep "gcc"
+      SPEC: "%gcc@4.9.3"
+      SPEC: "%gcc@6.1.0"
+      SPEC: "%gcc@7.1.0"
+      SPEC: "%gcc@7.3.0"
+      SPEC: "%gcc@8.1.0"
+
+We now have a list of the specs vetted on ``quartz``/``toss_3_x86_64_ib``.
+
+.. note::
+  In practice, one should check if the job is not *allowed to fail*, or even deactivated.
+
+MacOS case
+^^^^^^^^^^
+
+In CHAI, the Spack configuration for MacOS contains the default compilers depending on the OS version (`compilers.yaml`), and a commented section to illustrate how to add `CMake` as an external package. You may install CMake with homebrew, for example.
+
+
+Using Uberenv to generate the host-config file
+----------------------------------------------
+
+We have seen that we can safely use `gcc@8.1.0` on quartz. Let us ask for the default configuration first, and then ask for RAJA support and link to develop version of RAJA:
+
+.. code-block:: bash
+
+  $ python scripts/uberenv/uberenv.py --spec="%clang@9.0.0"
+  $ python scripts/uberenv/uberenv.py --spec="%clang@9.0.0+raja ^raja@develop"
+
+Each will generate a CMake cache file, e.g.:
+
+.. code-block:: bash
+
+  hc-quartz-toss_3_x86_64_ib-clang@9.0.0-fjcjwd6ec3uen5rh6msdqujydsj74ubf.cmake
+
+Using host-config files to build CHAI
+-------------------------------------
+
+.. code-block:: bash
+
+  $ mkdir build && cd build
+  $ cmake -C <path_to>/<host-config>.cmake ..
+  $ cmake --build -j .
+  $ ctest --output-on-failure -T test
+
+It is also possible to use this configuration with the CI script outside of CI:
+
+.. code-block:: bash
+
+  $ HOST_CONFIG=<path_to>/<host-config>.cmake scripts/gitlab/build_and_test.sh
+
+Testing new dependencies versions
+---------------------------------
+
+CHAI depends on Umpire, and optionally CHAI. Testing with newer versions of both is made straightforward with Uberenv and Spack:
+
+* ``$ python scripts/uberenv/uberenv.py --spec=%clang@9.0.0 ^umpire@develop``
+* ``$ python scripts/uberenv/uberenv.py --spec=%clang@9.0.0+raja ^raja@develop``
+
+Those commands will install respectively `umpire@develop` and `raja@develop` locally, and generate host-config files with the corresponding paths.
+
+Again, the CI script can be used directly to install, build and test in one command:
+
+.. code-block:: bash
+
+  $ SPEC="%clang@9.0.0 ^umpire@develop" scripts/gitlab/build_and_test.sh

docs/sphinx/developer_guide.rst

@@ -4,93 +4,12 @@
 Developer Guide
 ===============
 
-Generating CHAI host-config files
-===================================
+This section aims at gathering information useful to the developer.
 
-.. note::
-This mechanism will generate a cmake configuration file that reproduces the configuration `Spack <https://github.com/spack/spack>` would have generated in the same context. It will contain all the information necessary to build CHAI with the described toolchain.
+In particular, the local build scenarios as well as CI testing will be discussed here.
 
-In particular, the host config file will setup:
-* flags corresponding with the target required (Release, Debug).
-* compilers path, and other toolkits (cuda if required), etc.
-* paths to installed dependencies.
+.. toctree::
+   :maxdepth: 1
 
-This provides an easy way to build CHAI based on `Spack <https://github.com/spack/spack>` itself driven by `Uberenv <https://github.com/LLNL/uberenv>`_.
-
-Uberenv role
-------------
-
-Uberenv helps by doing the following:
-
-* Pulls a blessed version of Spack locally
-* If you are on a known operating system (like TOSS3), we have defined compilers and system packages so you don't have to rebuild the world (CMake typically in CHAI).
-* Overrides CHAI Spack packages with the local one if it exists. (see ``scripts/uberenv/packages``).
-* Covers both dependencies and project build in one command.
-
-Uberenv will create a directory ``uberenv_libs`` containing a Spack instance with the required CHAI dependencies installed. It then generates a host-config file (``<config_dependent_name>.cmake``) at the root of CHAI repository.
-
-Using Uberenv to generate the host-config file
-----------------------------------------------
-
-.. code-block:: bash
-
-  $ python scripts/uberenv/uberenv.py
-
-.. note::
-  On LC machines, it is good practice to do the build step in parallel on a compute node. Here is an example command: ``srun -ppdebug -N1 --exclusive python scripts/uberenv/uberenv.py``
-
-Unless otherwise specified Spack will default to a compiler. It is recommended to specify which compiler to use: add the compiler spec to the ``--spec`` Uberenv command line option.
-
-On blessed systems, compiler specs can be found in the Spack compiler files in our repository: ``scripts/uberenv/spack_configs/<System type>/compilers.yaml``.
-
-Some examples uberenv options:
-
-* ``--spec=%clang@4.0.0``
-* ``--spec=%clang@4.0.0+cuda``
-* ``--prefix=<Path to uberenv build directory (defaults to ./uberenv_libs)>``
-
-It is also possible to use the CI script outside of CI:
-
-.. code-block:: bash
-
-  $ SPEC="%clang@9.0.0 +cuda" scripts/gitlab/build_and_test.sh --deps-only
-
-Building dependencies can take a long time. If you already have a Spack instance you would like to reuse (in supplement of the local one managed by Uberenv), you can do so changing the uberenv command as follow:
-
-.. code-block:: bash
-
-  $ python scripts/uberenv/uberenv.py --upstream=<path_to_my_spack>/opt/spack
-
-Using host-config files to build CHAI
--------------------------------------
-
-When a host-config file exists for the desired machine and toolchain, it can easily be used in the CMake build process:
-
-.. code-block:: bash
-
-  $ mkdir build && cd build
-  $ cmake -C  <path_to>/lassen-blueos_3_ppc64le_ib_p9-clang@9.0.0.cmake ..
-  $ cmake --build -j .
-  $ ctest --output-on-failure -T test
-
-It is also possible to use the CI script outside of CI:
-
-.. code-block:: bash
-
-  $ HOST_CONFIG=<path_to>/<host-config>.cmake scripts/gitlab/build_and_test.sh
-
-Testing new dependencies versions
----------------------------------
-
-CHAI depends on Umpire, and optionally RAJA. Testing with newer versions of both is made straightforward with Uberenv and Spack:
-
-* ``$ python scripts/uberenv/uberenv.py --spec=%clang@9.0.0 ^umpire@develop``
-* ``$ python scripts/uberenv/uberenv.py --spec=%clang@9.0.0+raja ^raja@develop``
-
-Those commands will install respectively `umpire@develop` and `raja@develop` locally, and generate host-config files with the corresponding paths.
-
-Again, the CI script can be used directly to install, build and test in one command:
-
-.. code-block:: bash
-
-  $ SPEC="%clang@9.0.0 ^umpire@develop" scripts/gitlab/build_and_test.sh
+   developer/ci.rst
+   developer/uberenv.rst

scripts/make_release_tarball.sh

@@ -7,7 +7,7 @@
 ##############################################################################
 
 TAR_CMD=gtar
-VERSION=2.2.2
+VERSION=2.3.0
 
 git archive --prefix=chai-${VERSION}/ -o chai-${VERSION}.tar HEAD 2> /dev/null
 

src/chai/ArrayManager.cpp

@@ -158,6 +158,12 @@ void * ArrayManager::frontOfAllocation(void * pointer) {
 
 void ArrayManager::setExecutionSpace(ExecutionSpace space)
 {
+#if defined(CHAI_ENABLE_GPU_SIMULATION_MODE)
+   if (isGPUSimMode()) {
+      space = chai::GPU;
+   }
+#endif
+
   CHAI_LOG(Debug, "Setting execution space to " << space);
 
   if (chai::GPU == space) {

src/chai/ArrayManager.hpp

@@ -414,7 +414,19 @@ class ArrayManager
   /*!
    * \brief synchronize the device if there hasn't been a synchronize since the last kernel
    */
-  bool syncIfNeeded();
+  CHAISHAREDDLL_API bool syncIfNeeded();
+
+#if defined(CHAI_ENABLE_GPU_SIMULATION_MODE)
+  /*!
+   * \brief Turn the GPU simulation mode on or off.
+   */
+  void setGPUSimMode(bool gpuSimMode) { m_gpu_sim_mode = gpuSimMode; }
+
+  /*!
+   * \brief Return true if GPU simulation mode is on, false otherwise.
+   */
+  bool isGPUSimMode() { return m_gpu_sim_mode; }
+#endif
 
   /*!
    * \brief Evicts the data in the given space.
@@ -523,6 +535,14 @@ class ArrayManager
    * GPU context
    */
   bool m_synced_since_last_kernel = false;
+
+#if defined(CHAI_ENABLE_GPU_SIMULATION_MODE)
+  /*!
+   * Used by the RAJA plugin to determine whether the execution space should be
+   * CPU or GPU.
+   */
+  bool m_gpu_sim_mode = false;
+#endif
 };
 
 }  // end of namespace chai

src/chai/ManagedArray.hpp

@@ -399,7 +399,7 @@ class ManagedArray : public CHAICopyable
 #if !defined(CHAI_DEVICE_COMPILE)
   // if we can, ensure elems is based off the pointer_record size to protect against
   // casting leading to incorrect size info in m_elems.
-  if (m_pointer_record != nullptr) {
+  if (m_pointer_record != nullptr && !m_is_slice) {
      m_elems = m_pointer_record->m_size / sizeof(T);
   }
 #endif

src/chai/ManagedArray.inl

@@ -111,7 +111,7 @@ CHAI_HOST_DEVICE ManagedArray<T>::ManagedArray(ManagedArray const& other):
 #if !defined(CHAI_DEVICE_COMPILE)
   if (m_active_base_pointer || m_elems > 0 ) {
      // we only update m_elems if we are not null and we have a pointer record
-     if (m_pointer_record) {
+     if (m_pointer_record && !m_is_slice) {
         m_elems = m_pointer_record->m_size/sizeof(T);
      }
      move(m_resource_manager->getExecutionSpace());

src/chai/ManagedArrayView.hpp

@@ -28,6 +28,14 @@ using TypedManagedArrayView = RAJA::TypedViewBase<ValueType,
                                             LayoutType,
                                             IndexTypes...>;
 
+template <typename ValueType, typename LayoutType, RAJA::Index_type P2Pidx = 0>
+using ManagedArrayMultiView =
+    RAJA::MultiView<ValueType,
+                    LayoutType,
+                    P2Pidx,
+                    chai::ManagedArray<ValueType> *,
+                    chai::ManagedArray<camp::type::cv::rem<ValueType>> *>;
+
 } // end of namespace chai
 
 #endif // defined(CHAI_ENABLE_RAJA_PLUGIN)

src/chai/RajaExecutionSpacePlugin.cpp

@@ -12,14 +12,17 @@
 
 namespace chai {
 
-RajaExecutionSpacePlugin::RajaExecutionSpacePlugin() :
-  m_arraymanager(chai::ArrayManager::getInstance())
+RajaExecutionSpacePlugin::RajaExecutionSpacePlugin()
 {
 }
 
 void
 RajaExecutionSpacePlugin::preCapture(const RAJA::util::PluginContext& p)
 {
+  if (!m_arraymanager) {
+    m_arraymanager = chai::ArrayManager::getInstance();
+  }
+
   switch (p.platform) {
     case RAJA::Platform::host:
       m_arraymanager->setExecutionSpace(chai::CPU); break;

src/chai/RajaExecutionSpacePlugin.hpp

@@ -24,7 +24,7 @@ class RajaExecutionSpacePlugin :
     void postCapture(const RAJA::util::PluginContext& p) override;
 
   private:
-    chai::ArrayManager* m_arraymanager;
+    chai::ArrayManager* m_arraymanager{nullptr};
 };
 
 void linkRajaPlugin();