Simplify API, only consider order of ranks. No global indices.

include/ipc_debug.h

@@ -33,29 +33,20 @@ void debug_ipc_assert_equal_array(void *value, size_t size);
 void debug_ipc_assert_source_location(const char *source_file, const long int line_number);
 
 
-void debug_ipc_mpi_set_data_distribution(int start_index, uint64_t local_length);
+/**
+ * Debug IPC also allows comparing across multiple MPI ranks with differing cluster sizes.
+ * This method can be used to define the order of ranks.
+ */
+void debug_ipc_mpi_set_data_distribution(int start_index);
 
 /**
  * Compare two MPI distributed arrays between processes.
- * array_length denotes the number of double elements in the array, not the number of bytes.
+ * array_length denotes the local number of double elements in the array, not the number of bytes.
  *
  * The data distribution must be given through `debug_ipc_mpi_set_data_distribution` before calling
  * this method.
- *
- * Span is an additional parameter that acts as a multiplier on the assigned indices given by the data distribution.
- * For example, if our data distribution looks like this with three doubles assigned per global index:
- *                      Rank 0                  Rank 1               Rank 2
- *              ┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐
- *              │                   │ │                   │ │                   │
- *              │  A      B      C  │ │  A      B      C  │ │  A      B      C  │
- *              │                   │ │                   │ │                   │
- *              └───────────────────┘ └───────────────────┘ └───────────────────┘
- *  global index  0                     1                      2
- *
- *
- * …we would have to specify a span of 3, because each global index spans across 3 doubles.
  */
-void debug_ipc_assert_equal_mpi_double_array(double *array, size_t array_length, int span);
+void debug_ipc_assert_equal_mpi_double_array(double *array, size_t array_length);
 
 #ifdef __cplusplus
 }

src/ipc_debug.cpp

@@ -90,7 +90,7 @@ void debug_ipc_assert_equal(T value) {
 
         if (other_value != value) {
             std::cout << "[IPCDBG] Assertion failed!"
-                      << " Master has " << value << " but client has " << other_value;
+                      << " Root has " << value << " but client has " << other_value;
             print_backtrace();
             std::cout << "Entering endless loop, attach debugger to PID " << getpid();
             fflush(stdout);
@@ -231,87 +231,55 @@ void print_backtrace()
     p.print(st);
 }
 
-void debug_ipc_mpi_set_data_distribution(int start_index, uint64_t local_length) {
+void debug_ipc_mpi_set_data_distribution(int start_index) {
     if (rank == 0) {
         start_indices_map.clear();
         start_indices_list.resize(cluster_size + 1);
-        recv_counts.resize(cluster_size);
     }
 
-    unsigned long total_length = 0UL;
+    int my_start_index = start_index;
+    MPI_Gather(&my_start_index, 1, MPI_INT,
+            start_indices_list.data(), 1, MPI_INT,
+            0, MPI_COMM_WORLD);
 
-    unsigned long length = local_length;
-    MPI_Reduce(&length, &total_length, 1, MPI_UNSIGNED_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-    MPI_Gather(&start_index, 1, MPI_INT,
-            start_indices_list.data(), 1, MPI_INT, 0,
-            MPI_COMM_WORLD);
-
-    if (rank != 0) {
-        // Only root needs to now the sendcounts and displacements
-        return;
-    }
-
-
-    start_indices_map[total_length] = -1; // sentinel element
-    start_indices_list[cluster_size] = total_length;
-
-    for (int i = 0; i < cluster_size; i++) {
-        start_indices_map[start_indices_list[i]] = i;
-    }
-
-    // Calculate recv_counts
-    for (const auto& [start_index, m_rank] : start_indices_map) {
-        // Exit loop on sentinel element
-        if (m_rank == -1) {
-            break;
+    if (rank == 0) {
+        for (int i = 0; i < cluster_size; i++) {
+            start_indices_map[start_indices_list[i]] = i;
         }
-
-        const uint64_t next_index = start_indices_map.upper_bound(start_index)->first;
-        const uint64_t segment_length = next_index - start_index;
-        recv_counts[m_rank] = segment_length;
-        printf("Recv counts [%i] = %lu\n", m_rank, segment_length);
     }
-
-    uint64_t total_recvs = std::accumulate(recv_counts.begin(), recv_counts.end(), 0);
-    printf("total recv = %lu, total length = %lu\n", total_recvs, total_length);
-    assert(total_length == total_recvs);
-    debug_ipc_assert_equal<uint64_t>(total_length);
 }
 
 
 
 
-void debug_ipc_assert_equal_mpi_double_array(double *array, size_t array_length, int span) {
+void debug_ipc_assert_equal_mpi_double_array(double *array, size_t array_length) {
+    std::vector<int> recv_counts(cluster_size);
+    std::vector<int> displacements(cluster_size);
+    int local_array_length = static_cast<int>(array_length);
 
-    if (rank == 0) {
-        std::vector<int> actual_recv_counts(recv_counts);
-        std::vector<int> displacements(recv_counts);
+    // Gather local lengths
+    MPI_Gather(&local_array_length, 1, MPI_INT,
+            recv_counts.data(), 1, MPI_INT,
+            0, MPI_COMM_WORLD);
 
-        size_t total_length = span * start_indices_list[cluster_size];
-        float_buffer.resize(total_length); // Make sure we have enough space for all values
-        assert(actual_recv_counts.size() == static_cast<unsigned>(cluster_size));
-        assert(start_indices_map.size() == static_cast<unsigned>(cluster_size + 1));
-
-        for (unsigned int i = 0; i < recv_counts.size(); i++) {
-            actual_recv_counts[i] *= span;
+    if (rank == 0) {
+        // Setup displacements and receive counts
+        int global_index = 0;
+        // Iterate over start indices from small to large
+        for (const auto& [start_index, m_rank] : start_indices_map) {
+            displacements[m_rank] = global_index;
+            global_index += recv_counts[m_rank];
         }
+        float_buffer.resize(global_index); // Make sure buffer is appropriately sized.
+    }
 
-        for (unsigned int i = 0; i < recv_counts.size(); i++) {
-            displacements[i] *= span;
-        }
+    MPI_Gatherv(array, array_length, MPI_DOUBLE,
+            float_buffer.data(), // recvbuf
+            recv_counts.data(),  // recv counts
+            displacements.data(), // displacements
+            MPI_DOUBLE, 0, MPI_COMM_WORLD);
 
-        MPI_Gatherv(array, array_length * span, MPI_DOUBLE,
-                float_buffer.data(), // recvbuf
-                actual_recv_counts.data(),  // recv counts
-                start_indices_list.data(), // displacements
-                MPI_DOUBLE, 0, MPI_COMM_WORLD);
+    if (rank == 0) {
         debug_ipc_assert_equal_vector(float_buffer);
-    } else {
-        // Gather all data at the root.
-        MPI_Gatherv(array, array_length * span, MPI_DOUBLE,
-                nullptr, // recvbuf
-                nullptr,  // recv counts
-                nullptr, // displacements
-                MPI_DOUBLE, 0, MPI_COMM_WORLD);
     }
 }

src/mpi_test.cpp

@@ -57,10 +57,10 @@ int main(int argc, char **argv) {
 
     // Setup IPC DEBUG
     debug_ipc_init();
-    debug_ipc_mpi_set_data_distribution(start_index, characters_per_rank);
+    debug_ipc_mpi_set_data_distribution(start_index);
 
     // Make actual assertion
-    debug_ipc_assert_equal_mpi_double_array(arr.data(), characters_per_rank, 1);
+    debug_ipc_assert_equal_mpi_double_array(arr.data(), characters_per_rank);
 
     MPI_Barrier(MPI_COMM_WORLD);
     MPI_Finalize();