From 9177df9456a51e3acc82e60143ed1c862cc3479f Mon Sep 17 00:00:00 2001 From: IgorBaratta Date: Thu, 11 May 2023 16:19:06 +0100 Subject: [PATCH 1/4] add function to compute IndexMap imbalance --- cpp/dolfinx/common/IndexMap.cpp | 31 +++++++++++++++++++++++++++++++ cpp/dolfinx/common/IndexMap.h | 14 ++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 67517616eff..9b958a9a232 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -948,3 +948,34 @@ const std::vector& IndexMap::dest() const noexcept { return _dest; } //----------------------------------------------------------------------------- bool IndexMap::overlapped() const noexcept { return _overlapping; } //----------------------------------------------------------------------------- +std::array IndexMap::imbalance() const noexcept +{ + std::array imbalance{-1., -1.}; + std::array max_count; + std::array local_data + = {_local_range[1] - _local_range[0], _ghosts.size()}; + + // Find the maximum number of owned indices and the maximum number of ghost + // indices across all processes. + MPI_Allreduce(local_data.data(), max_count.data(), 2, + dolfinx::MPI::mpi_type(), MPI_MAX, _comm.comm()); + + std::int32_t total_num_ghosts = 0; + MPI_Allreduce(&local_data[1], &total_num_ghosts, 1, + dolfinx::MPI::mpi_type(), MPI_SUM, _comm.comm()); + + // Compute the average number of owned and ghost indices per process. + int comm_size = dolfinx::MPI::size(_comm.comm()); + double avg_owned = static_cast(_size_global) / comm_size; + double avg_ghosts = static_cast(total_num_ghosts) / comm_size; + + // Compute the imbalance by dividing the maximum number of indices by the + // corresponding average. + if (avg_owned > 0) + imbalance[0] = max_count[0] / avg_owned; + if (avg_ghosts > 0) + imbalance[1] = max_count[1] / avg_ghosts; + + return imbalance; +} +//----------------------------------------------------------------------------- diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 018ba279033..21f1baad447 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -228,6 +228,20 @@ class IndexMap /// false. bool overlapped() const noexcept; + /// @brief Returns the imbalance of the current IndexMap. + /// + /// The imbalance is a measure of load balancing across all processes, defined + /// as the maximum number of indices on any process divided by the average + /// number of indices per process. This function calculates the imbalance + /// separately for owned indices and ghost indices and returns them as a + /// std::array. + /// + /// @note Collective + /// + /// @return An array containing the imbalance in owned indices + /// (first element) and the imbalance in ghost indices (second element). + std::array imbalance() const noexcept; + private: // Range of indices (global) owned by this process std::array _local_range; From 32370dc90f34113cf2eb8a52bd1828ed474636de Mon Sep 17 00:00:00 2001 From: IgorBaratta Date: Thu, 11 May 2023 16:24:50 +0100 Subject: [PATCH 2/4] fix cast issue --- cpp/dolfinx/common/IndexMap.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 9b958a9a232..47f30ac4379 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -952,16 +952,17 @@ std::array IndexMap::imbalance() const noexcept { std::array imbalance{-1., -1.}; std::array max_count; - std::array local_data - = {_local_range[1] - _local_range[0], _ghosts.size()}; + std::array local_sizes + = {_local_range[1] - _local_range[0], + static_cast(_ghosts.size())}; // Find the maximum number of owned indices and the maximum number of ghost // indices across all processes. - MPI_Allreduce(local_data.data(), max_count.data(), 2, + MPI_Allreduce(local_sizes.data(), max_count.data(), 2, dolfinx::MPI::mpi_type(), MPI_MAX, _comm.comm()); std::int32_t total_num_ghosts = 0; - MPI_Allreduce(&local_data[1], &total_num_ghosts, 1, + MPI_Allreduce(&local_sizes[1], &total_num_ghosts, 1, dolfinx::MPI::mpi_type(), MPI_SUM, _comm.comm()); // Compute the average number of owned and ghost indices per process. From ab72ad14b24df700e4193f1052fd5e31affb2d5d Mon Sep 17 00:00:00 2001 From: IgorBaratta Date: Thu, 11 May 2023 16:30:36 +0100 Subject: [PATCH 3/4] minor update --- cpp/dolfinx/common/IndexMap.cpp | 4 ++-- cpp/dolfinx/common/IndexMap.h | 2 +- python/dolfinx/wrappers/common.cpp | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.cpp b/cpp/dolfinx/common/IndexMap.cpp index 47f30ac4379..f0acb93bc1d 100644 --- a/cpp/dolfinx/common/IndexMap.cpp +++ b/cpp/dolfinx/common/IndexMap.cpp @@ -948,12 +948,12 @@ const std::vector& IndexMap::dest() const noexcept { return _dest; } //----------------------------------------------------------------------------- bool IndexMap::overlapped() const noexcept { return _overlapping; } //----------------------------------------------------------------------------- -std::array IndexMap::imbalance() const noexcept +std::array IndexMap::imbalance() const { std::array imbalance{-1., -1.}; std::array max_count; std::array local_sizes - = {_local_range[1] - _local_range[0], + = {static_cast(_local_range[1] - _local_range[0]), static_cast(_ghosts.size())}; // Find the maximum number of owned indices and the maximum number of ghost diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 21f1baad447..4773599819a 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -240,7 +240,7 @@ class IndexMap /// /// @return An array containing the imbalance in owned indices /// (first element) and the imbalance in ghost indices (second element). - std::array imbalance() const noexcept; + std::array imbalance() const; private: // Range of indices (global) owned by this process diff --git a/python/dolfinx/wrappers/common.cpp b/python/dolfinx/wrappers/common.cpp index 7c5f6f32a3a..dfb4d5b1075 100644 --- a/python/dolfinx/wrappers/common.cpp +++ b/python/dolfinx/wrappers/common.cpp @@ -98,6 +98,8 @@ void common(py::module& m) .def_property_readonly("local_range", &dolfinx::common::IndexMap::local_range, "Range of indices owned by this map") + .def_property_readonly("imbalance", &dolfinx::common::IndexMap::imbalance, + "Imbalance of the current IndexMap.") .def_property_readonly( "ghosts", [](const dolfinx::common::IndexMap& self) From 9ba2c5bbe9312eb134cd8f46cf2ad83ff688841b Mon Sep 17 00:00:00 2001 From: IgorBaratta Date: Thu, 11 May 2023 17:14:58 +0100 Subject: [PATCH 4/4] update documentation --- cpp/dolfinx/common/IndexMap.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/dolfinx/common/IndexMap.h b/cpp/dolfinx/common/IndexMap.h index 4773599819a..9e4a0ee131a 100644 --- a/cpp/dolfinx/common/IndexMap.h +++ b/cpp/dolfinx/common/IndexMap.h @@ -234,9 +234,11 @@ class IndexMap /// as the maximum number of indices on any process divided by the average /// number of indices per process. This function calculates the imbalance /// separately for owned indices and ghost indices and returns them as a - /// std::array. + /// std::array. If the total number of owned or ghost indices is + /// zero, the respective entry in the array is set to -1. /// - /// @note Collective + /// @note This is a collective operation and must be called by all processes + /// in the communicator associated with the IndexMap. /// /// @return An array containing the imbalance in owned indices /// (first element) and the imbalance in ghost indices (second element).