Add response file, make client stop when assertion fails

src/ipc_debug.cpp

@@ -19,6 +19,7 @@
 
 bool debug_ipc_is_root = false;
 FILE *debug_ipc_file = nullptr;
+FILE *debug_ipc_response_file = nullptr;
 
 int rank, cluster_size;
 std::map<uint64_t, int> start_indices_map; // maps indices to MPI ranks
@@ -43,6 +44,7 @@ void debug_ipc_init() {
     if (rank == 0) {
         const char *root_env = std::getenv("IPC_DEBUG_ROOT");
         const char *file_env = std::getenv("IPC_DEBUG_FILE");
+        const char *response_file_env = std::getenv("IPC_DEBUG_RESPONSE_FILE");
         
         if (file_env == nullptr) {
             return;
@@ -66,12 +68,50 @@ void debug_ipc_init() {
             exit(-1);
         }
 
+        // Open response file for the back channel
+        if (response_file_env != nullptr) {
+            if (debug_ipc_is_root) {
+                debug_ipc_response_file = fopen(response_file_env, "w");
+            } else {
+                debug_ipc_response_file = fopen(response_file_env, "r");
+            }
+
+            if (debug_ipc_file == nullptr) {
+                printf("[IPCDBG] Error, could not open named pipe for responses: %s", strerror(errno));
+                exit(-1);
+            }
+
+        }
 
         buffer.resize(INITIAL_BUFFER_SIZE);
     }
 
 }
 
+/** Communicate assertion result to client and possibly hang if assertion failed
+ */
+inline void handle_assertion(bool assertion_status = true) {
+    fflush(debug_ipc_file); // make sure the comparison is done
+    if (!debug_ipc_response_file)
+        return;
+
+    char res = assertion_status ? 1 : 0;
+    if (debug_ipc_is_root) {
+        fwrite(&res, 1, 1, debug_ipc_response_file);
+        fflush(debug_ipc_response_file);
+    } else {
+        fread(&res, 1, 1, debug_ipc_response_file);
+    }
+
+    if (res == 0) {
+        std::cout << "[IPCDBG] Entering endless loop, attach debugger to PID " << getpid();
+        fflush(stdout);
+        while (1) {
+            sleep(1);
+        }
+    }
+}
+
 template<typename T>
 void debug_ipc_assert_equal(T value) {
     if (debug_ipc_file == nullptr) {
@@ -90,18 +130,14 @@ void debug_ipc_assert_equal(T value) {
 
         if (other_value != value) {
             std::cout << "[IPCDBG] Assertion failed!"
-                      << " Root has " << value << " but client has " << other_value;
+                      << " Root has " << value << " but client has " << other_value << std::endl;
             print_backtrace();
-            std::cout << "Entering endless loop, attach debugger to PID " << getpid();
-            fflush(stdout);
-            while (1) {
-                sleep(1);
-            }
         } else {
 #ifdef TRACE
             std::cout << "[IPCDBG] Assertion passed, value = " << value << std::endl;
 #endif
         }
+        handle_assertion(other_value == value);
     } else {
         size_t written = fwrite(&value, expected_size, 1, debug_ipc_file);
 
@@ -109,6 +145,7 @@ void debug_ipc_assert_equal(T value) {
             printf("[IPCDBG] Could not write enough bytes. Error: %s\n", strerror(errno));
             exit(-1);
         }
+        handle_assertion();
     }
 }
 
@@ -134,20 +171,20 @@ void debug_ipc_assert_equal_vector(std::vector<T> value) {
         T *local_array = value.data();
         T *other_array = reinterpret_cast<T *>(buffer.data());
 
+        bool items_equal = true;
         for (size_t i = 0; i < value.size(); ++i) {
             if (local_array[i] != other_array[i]) {
                 std::cout << "[IPCDBG] Assertion failed in vector  at index " << i 
                     << ". Root has " << local_array[i] << " but client has " << other_array[i] << std::endl;
                 print_backtrace();
-                printf("Entering endless loop, attach debugger to PID %i \n", getpid());
-                fflush(stdout);
-                while (1) {
-                    sleep(1);
-                }
+                items_equal = false;
+                break;
             }
         }
+        handle_assertion(items_equal);
     } else {
         fwrite(value.data(), 1, array_byte_length, debug_ipc_file);
+        handle_assertion();
     }
 }
 
@@ -194,16 +231,16 @@ void debug_ipc_assert_equal_array(void *value, size_t size) {
             exit(-1);
         }
 
+        bool items_equal = true;
         for (size_t i = 0; i < size; i++) {
             if (array[i] != other_array[i]) {
                 printf("[IPCDBG] Assertion failed in byte %lu!\n", i);
                 print_backtrace();
-                printf("Entering endless loop, attach debugger to PID %i \n", getpid());
-                while (1) {
-                    sleep(1);
-                }
+                items_equal = false;
+                break;
             }
         }
+        handle_assertion(items_equal);
     } else {
         size_t written = fwrite(value, 1, size, debug_ipc_file);
 
@@ -211,6 +248,8 @@ void debug_ipc_assert_equal_array(void *value, size_t size) {
             printf("[IPCDBG] Could not write enough bytes. Error: %s\n", strerror(errno));
             exit(-1);
         }
+
+        handle_assertion();
     }
 }