From 49a7e94fb039616881eabfbb3425697e6f0026e2 Mon Sep 17 00:00:00 2001 From: Alwalid Salama Date: Thu, 16 Jul 2026 01:01:36 +0200 Subject: [PATCH 1/5] tests: skip all coroutine tests on Windows The guard used the invalid CMake `in` operator (newer CMake on the clangarm64 runner rejects it, failing configuration). Signed-off-by: Alwalid Salama --- tests/qbox/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/qbox/CMakeLists.txt b/tests/qbox/CMakeLists.txt index 0d2fa9b6..e2743eb0 100644 --- a/tests/qbox/CMakeLists.txt +++ b/tests/qbox/CMakeLists.txt @@ -155,7 +155,7 @@ function(qbox_add_cpu_test target timeout_s) # On Windows, coroutine mode is only supported with 1 or 2 CPUs set(WINDOWS_COROUTINE_ALLOWED_NUM_CPU 1 2) - if (("${threading}" STREQUAL "COROUTINE") AND (NOT (${num_cpu} in WINDOWS_COROUTINE_ALLOWED_NUM_CPU) ) AND (WIN32)) + if (("${threading}" STREQUAL "COROUTINE") AND (WIN32)) continue() endif() From ef78ec776405cd1fbf20e99084f29ec049432c6c Mon Sep 17 00:00:00 2001 From: Alwalid Salama Date: Wed, 15 Jul 2026 16:12:56 +0200 Subject: [PATCH 2/5] cpu.h: fix MCIPS teardown UAF and lost reset wakeup MCIPS vCPUs are free-running MTTCG threads. McipsSync had no on_destroy(), so after sc_stop() a vCPU could still be in tcg_cpu_exec() and touch the freed QemuInstance -> intermittent SIGSEGV/hang. Add on_destroy() to join the vCPU (cpu_remove_sync under the BQL) before teardown. reset_cb()'s hold loop kicked the vCPU without the BQL; qemu_cpu_kick() broadcasts halt_cond unlocked, so the kick could be lost against a vCPU entering qemu_cond_wait() (WFI), and the queued reset job never ran -> hang. Kick under the BQL instead. Signed-off-by: Alwalid Salama --- qemu-components/common/include/cpu.h | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/qemu-components/common/include/cpu.h b/qemu-components/common/include/cpu.h index 6ee899e2..0c9418e2 100644 --- a/qemu-components/common/include/cpu.h +++ b/qemu-components/common/include/cpu.h @@ -130,6 +130,7 @@ class QemuCpu : public QemuDevice, public QemuInitiatorIface using CpuTimeSyncStrategy::CpuTimeSyncStrategy; void on_end_of_elaboration() override; + void on_destroy() override; }; private: @@ -669,7 +670,17 @@ class QemuCpu : public QemuDevice, public QemuInitiatorIface if (m_resetting == none) return; // dont finish a finished reset! while (m_resetting == start_reset) { SCP_WARN(())("Hold reset"); + /* + * Kick under the BQL. qemu_cpu_kick() broadcasts the vCPU's + * halt_cond without the BQL, so a bare kick can be lost against a + * vCPU entering qemu_cond_wait(halt_cond) (e.g. sleeping in WFI) + * and the queued reset(true) job is never run. Holding the BQL + * serialises against that. Release it before waiting, the vCPU + * needs it to run the job. + */ + m_inst.get().lock_iothread(); m_cpu.kick(); // without this kick, async_safe_run may not be called (QEMU race) + m_inst.get().unlock_iothread(); sc_core::wait(m_start_reset_done_ev); } m_inst.get().lock_iothread(); @@ -907,4 +918,23 @@ inline void QemuCpu::McipsSync::on_end_of_elaboration() } } +inline void QemuCpu::McipsSync::on_destroy() +{ + /* + * In MCIPS mode the vCPU is a free-running MTTCG thread with no QK/deadline + * timer gating it to SystemC. end_of_simulation() only requests halt+unplug + * asynchronously, so after sc_stop() the thread can still be in tcg_cpu_exec() + * and touch the QemuInstance after it is freed -> use-after-free. By now the + * simulation has stopped and runonsysc jobs are cancelled, so it is safe to + * synchronously join the vCPU thread before the QemuInstance is destroyed. + */ + if (!m_qemu_cpu.m_started || m_qemu_cpu.m_coroutines || !m_qemu_cpu.m_cpu.valid()) { + return; + } + + m_qemu_cpu.m_inst.get().lock_iothread(); + m_qemu_cpu.m_cpu.remove_sync(); // stop + unplug + join the vCPU thread + m_qemu_cpu.m_inst.get().unlock_iothread(); +} + #endif From 975116cae7179316503c46b06afa8f311ab3de5d Mon Sep 17 00:00:00 2001 From: Alwalid Salama Date: Fri, 17 Jul 2026 11:50:22 +0200 Subject: [PATCH 3/5] sync_window: defer attach/detach_suspending to the SystemC thread attach()/detach() ran SystemC's async_(at|de)tach_suspending() on the caller's thread (the QEMU vCPU thread in MCIPS). Those internals mutate the suspending-channel registry unlocked, racing the SystemC thread in async_suspend() and wedging the kernel. Like gs::async_event, defer the call to update() when invoked cross-thread. Signed-off-by: Alwalid Salama --- .../common/include/sync_window.h | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/systemc-components/common/include/sync_window.h b/systemc-components/common/include/sync_window.h index d6513c75..a1ed7935 100644 --- a/systemc-components/common/include/sync_window.h +++ b/systemc-components/common/include/sync_window.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #ifdef SC_HAS_OB_EVENT @@ -58,6 +59,8 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe sc_event m_update_ev; std::mutex m_mutex; sc_sync_policy policy; + std::thread::id m_tid; + enum class pending_state { none, attach, detach } m_attach_pending_state = pending_state::none; public: struct window { @@ -142,6 +145,41 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe } /* let stepper handle suspend/resume, must time notify */ m_update_ev.notify(sc_core::SC_ZERO_TIME); + + /* + * Apply any attach/detach requested from a foreign thread here, on the + * SystemC thread. SystemC's async_(at|de)tach_suspending() mutate the + * kernel's internal suspending-channel registry (a plain vector + bool) + * with NO locking of their own -- calling them directly from a QEMU + * vCPU thread races the SystemC thread's concurrent read of that state + * in async_suspend(), which can wedge the kernel in async_suspend() + * forever or miss a wakeup. + */ + switch (m_attach_pending_state) { + case pending_state::attach: + this->async_attach_suspending(); + break; + case pending_state::detach: + this->async_detach_suspending(); + break; + default: + break; + } + m_attach_pending_state = pending_state::none; + } + + /* Attach/detach the suspend registry. Safe from any thread: on the SystemC + * thread we mutate the (unlocked) registry directly; from a foreign thread we + * only record the request and let update() apply it on the SystemC thread. */ + void request_suspend(pending_state s) + { + if (m_tid == std::this_thread::get_id()) { + (s == pending_state::attach) ? async_attach_suspending() : async_detach_suspending(); + } else { + std::lock_guard lg(m_mutex); + m_attach_pending_state = s; + async_request_update(); + } } public: @@ -180,7 +218,7 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe */ void attach() { - async_attach_suspending(); + request_suspend(pending_state::attach); m_is_attached = true; } /** @@ -197,7 +235,7 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe { async_set_window({ now, sc_core::sc_max_time() }); // setting open window before detaching, since // async_set_window required an attached window. - async_detach_suspending(); + request_suspend(pending_state::detach); m_is_attached = false; } void bind(sc_sync_window* other) @@ -220,6 +258,7 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe } SC_CTOR (sc_sync_window) : m_window({sc_core::SC_ZERO_TIME, policy.quantum()}), m_is_attached(false) { + m_tid = std::this_thread::get_id(); SCP_TRACE(())("Constructor"); SC_METHOD(sweep_helper); dont_initialize(); From 30173e1f7f0eceb292cbc61aab989da1025586f5 Mon Sep 17 00:00:00 2001 From: Mahmoud Kamel Date: Tue, 14 Jul 2026 13:27:42 +0200 Subject: [PATCH 4/5] Package-lock: update libqemu to libqemu-v11.0-v0.11 - hw/timer/qct-qtimer: use 56-bit counter width - hw/timer/qct-qtimer: accept 64-bit MMIO accesses for hex_timer_ops - target/riscv: allow setting resetvec after realize - libqemu: build QEMU in parallel, generator-aware - ci: fix macOS failures due to untrusted quic homebrew tap Signed-off-by: Mahmoud Kamel --- package-lock.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.cmake b/package-lock.cmake index 497006ec..04e0e536 100644 --- a/package-lock.cmake +++ b/package-lock.cmake @@ -27,7 +27,7 @@ CPMDeclarePackage(SCP CPMDeclarePackage(qemu NAME libqemu GIT_REPOSITORY ${LIBQEMU_GIT} - GIT_TAG libqemu-v11.0-v0.10 + GIT_TAG libqemu-v11.0-v0.11 GIT_SUBMODULES CMakeLists.txt GIT_SHALLOW ON ) From 1e26ffde4cf6dead9e6458cd4f941defb7b7a8bd Mon Sep 17 00:00:00 2001 From: Alwalid Salama Date: Fri, 17 Jul 2026 17:37:41 +0200 Subject: [PATCH 5/5] draft: test trace Signed-off-by: Alwalid Salama --- .github/workflows/ubuntu-builds.yml | 7 +- qemu-components/common/include/mcips-plugin.h | 51 +++- .../common/include/async_event.h | 5 + .../common/include/gs_suspend_debug.h | 89 +++++++ .../common/include/sync_window.h | 11 +- tests/qbox/include/test/hang_watchdog.h | 238 ++++++++++++++++++ tests/qbox/include/test/test.h | 4 + 7 files changed, 402 insertions(+), 3 deletions(-) create mode 100644 systemc-components/common/include/gs_suspend_debug.h create mode 100644 tests/qbox/include/test/hang_watchdog.h diff --git a/.github/workflows/ubuntu-builds.yml b/.github/workflows/ubuntu-builds.yml index 3b854fb4..525528e3 100644 --- a/.github/workflows/ubuntu-builds.yml +++ b/.github/workflows/ubuntu-builds.yml @@ -27,4 +27,9 @@ jobs: run: cmake --build --preset "${{ matrix.preset }}" --parallel $(nproc) - name: Test - run: ctest --preset "${{ matrix.preset }}" + env: + # Debug: dump all-thread backtraces + MCIPS state and abort a hung test + # ~20s before ctest's 100s TIMEOUT, so intermittent mcips+MULTI deadlocks + # surface with a diagnosable log instead of an opaque timeout. + QBOX_HANG_WATCHDOG_SEC: "80" + run: ctest --preset "${{ matrix.preset }}" --output-on-failure diff --git a/qemu-components/common/include/mcips-plugin.h b/qemu-components/common/include/mcips-plugin.h index 54d4a106..3a8a5f3e 100644 --- a/qemu-components/common/include/mcips-plugin.h +++ b/qemu-components/common/include/mcips-plugin.h @@ -16,6 +16,9 @@ #include #include #include +#include +#include +#include class McipsPlugin : public LibQemuPlugin { @@ -89,6 +92,34 @@ class McipsPlugin : public LibQemuPlugin SC_METHOD(idle_tick_method); dont_initialize(); sensitive << m_idle_tick; + { + std::lock_guard lk(debug_registry_mutex()); + debug_registry().push_back(this); + } + } + + /* --- Debug hang-diagnosis support (see tests/.../hang_watchdog.h) -------- + * A process-wide registry of live plugins so an external watchdog can dump + * each instance's idle/window state when a run wedges. Read-only helpers; + * the dumper reads lock-free so it is safe to call while vCPU/SystemC + * threads are deadlocked. */ + static std::mutex& debug_registry_mutex() + { + static std::mutex m; + return m; + } + static std::vector& debug_registry() + { + static std::vector v; + return v; + } + void debug_dump_state() + { + auto* active = m_active_vcpu.load(std::memory_order_relaxed); + fprintf(stderr, "[MCIPS %s] active=%p attached=%d first_vcpu_init=%d qemu_time=%.9f shutdown=%d %s\n", name(), + (void*)active, (int)m_sync_sc.is_attached(), (int)m_first_vcpu_initialized.load(), + m_qemu_time.to_seconds(), (int)m_shutdown.load(), get_mcips_status_json().c_str()); + fflush(stderr); } /** @@ -143,7 +174,15 @@ class McipsPlugin : public LibQemuPlugin detach_sync_window(); } - ~McipsPlugin() { shutdown_cleanup(); } + ~McipsPlugin() + { + { + std::lock_guard lk(debug_registry_mutex()); + auto& v = debug_registry(); + v.erase(std::remove(v.begin(), v.end(), this), v.end()); + } + shutdown_cleanup(); + } /** @brief Called by SystemC at end of simulation; stops all QEMU callbacks. */ void end_of_simulation() override { shutdown_cleanup(); } @@ -808,4 +847,14 @@ class McipsPlugin : public LibQemuPlugin } }; +/* Weak hook picked up by the test hang-watchdog (tests/.../hang_watchdog.h) to + * dump every live plugin's idle/window state when a run wedges. Defined inline + * (vague linkage) so it exists wherever this header is compiled in, and is + * simply absent otherwise. */ +extern "C" inline void qbox_wd_dump_extra_state() +{ + std::lock_guard lk(McipsPlugin::debug_registry_mutex()); + for (auto* p : McipsPlugin::debug_registry()) p->debug_dump_state(); +} + #endif //_LIBQBOX_COMPONENTS_MCIPS_PLUGIN_H diff --git a/systemc-components/common/include/async_event.h b/systemc-components/common/include/async_event.h index 5b90815d..da56a8fc 100644 --- a/systemc-components/common/include/async_event.h +++ b/systemc-components/common/include/async_event.h @@ -16,6 +16,7 @@ #include #include +#include namespace gs { class async_event : public sc_core::sc_prim_channel, public sc_core::sc_event @@ -65,9 +66,11 @@ class async_event : public sc_core::sc_prim_channel, public sc_core::sc_event { #ifndef SC_HAS_ASYNC_ATTACH_SUSPENDING sc_core::async_attach_suspending(this); + gs_suspend_debug::on_attach(this, "async_event"); #else if (tid == std::this_thread::get_id()) { this->sc_core::sc_prim_channel::async_attach_suspending(); + gs_suspend_debug::on_attach(this, "async_event"); } else { mutex.lock(); attach_pending_state = attach; @@ -81,9 +84,11 @@ class async_event : public sc_core::sc_prim_channel, public sc_core::sc_event { #ifndef SC_HAS_ASYNC_ATTACH_SUSPENDING sc_core::async_detach_suspending(this); + gs_suspend_debug::on_detach(this); #else if (tid == std::this_thread::get_id()) { this->sc_core::sc_prim_channel::async_detach_suspending(); + gs_suspend_debug::on_detach(this); } else { mutex.lock(); attach_pending_state = detach; diff --git a/systemc-components/common/include/gs_suspend_debug.h b/systemc-components/common/include/gs_suspend_debug.h new file mode 100644 index 00000000..f0079b9f --- /dev/null +++ b/systemc-components/common/include/gs_suspend_debug.h @@ -0,0 +1,89 @@ +/* + * This file is part of libqbox + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. All Rights Reserved. + * + * SPDX-License-Identifier: BSD-3-Clause-Clear + * + * Debug-only registry of currently attached "suspending" channels + * (gs::async_event and sc_sync_window). SystemC's async_suspend() blocks while + * ANY prim_channel is attached-suspending, but it exposes no way to see which. + * When an mcips run wedges in async_suspend(), the hang watchdog dumps this + * registry (via the weak hook below) so we can see exactly which channel(s) are + * still holding the kernel, and where each was attached from. + * + * Linux-only (uses glibc backtrace); a no-op elsewhere. Enabled only when the + * watchdog is armed matters little -- the tracking itself is cheap (a map + * insert/erase under a mutex on attach/detach, which are not hot paths). + */ + +#ifndef GS_SUSPEND_DEBUG_H +#define GS_SUSPEND_DEBUG_H + +#if defined(__linux__) + +#include +#include +#include +#include +#include + +namespace gs_suspend_debug { + +struct Entry { + const char* kind; + void* frames[6]; + int nframes; +}; + +inline std::mutex& mtx() +{ + static std::mutex m; + return m; +} +inline std::map& reg() +{ + static std::map m; + return m; +} + +inline void on_attach(const void* ch, const char* kind) +{ + Entry e; + e.kind = kind; + e.nframes = backtrace(e.frames, 6); + std::lock_guard l(mtx()); + reg()[ch] = e; +} + +inline void on_detach(const void* ch) +{ + std::lock_guard l(mtx()); + reg().erase(ch); +} + +inline void dump() +{ + std::lock_guard l(mtx()); + std::fprintf(stderr, "[SUSPEND] %zu attached suspending channel(s):\n", reg().size()); + for (auto& kv : reg()) { + std::fprintf(stderr, " %s @ %p attached from:\n", kv.second.kind, kv.first); + if (kv.second.nframes > 0) backtrace_symbols_fd(kv.second.frames, kv.second.nframes, 2 /*stderr*/); + } + std::fflush(stderr); +} + +} // namespace gs_suspend_debug + +/* Weak hook picked up by the hang watchdog (tests/.../hang_watchdog.h). */ +extern "C" inline void qbox_wd_dump_suspend_channels() { gs_suspend_debug::dump(); } + +#else // !__linux__ + +namespace gs_suspend_debug { +inline void on_attach(const void*, const char*) {} +inline void on_detach(const void*) {} +} // namespace gs_suspend_debug + +#endif // __linux__ + +#endif // GS_SUSPEND_DEBUG_H diff --git a/systemc-components/common/include/sync_window.h b/systemc-components/common/include/sync_window.h index a1ed7935..7be0755c 100644 --- a/systemc-components/common/include/sync_window.h +++ b/systemc-components/common/include/sync_window.h @@ -12,6 +12,7 @@ #include #include #include +#include #ifdef SC_HAS_OB_EVENT #include @@ -158,9 +159,11 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe switch (m_attach_pending_state) { case pending_state::attach: this->async_attach_suspending(); + gs_suspend_debug::on_attach(this, "sync_window"); break; case pending_state::detach: this->async_detach_suspending(); + gs_suspend_debug::on_detach(this); break; default: break; @@ -174,7 +177,13 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe void request_suspend(pending_state s) { if (m_tid == std::this_thread::get_id()) { - (s == pending_state::attach) ? async_attach_suspending() : async_detach_suspending(); + if (s == pending_state::attach) { + async_attach_suspending(); + gs_suspend_debug::on_attach(this, "sync_window"); + } else { + async_detach_suspending(); + gs_suspend_debug::on_detach(this); + } } else { std::lock_guard lg(m_mutex); m_attach_pending_state = s; diff --git a/tests/qbox/include/test/hang_watchdog.h b/tests/qbox/include/test/hang_watchdog.h new file mode 100644 index 00000000..b3bd7e98 --- /dev/null +++ b/tests/qbox/include/test/hang_watchdog.h @@ -0,0 +1,238 @@ +/* + * This file is part of libqbox + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. All Rights Reserved. + * + * SPDX-License-Identifier: BSD-3-Clause-Clear + * + * Debug-only hang watchdog for the qbox CPU tests. + * + * ctest kills a timed-out test with SIGKILL, which leaves no diagnostic. When + * armed (env QBOX_HANG_WATCHDOG_SEC=), this starts a detached thread + * that, after the given number of seconds, dumps a backtrace of EVERY thread in + * the process to stderr and then aborts. Aborting a few seconds before ctest's + * own TIMEOUT means the intermittent mcips+MULTI deadlocks surface as an + * "aborted" test whose log pins where every thread is stuck, instead of an + * opaque timeout. It is a no-op unless the env var is set, so normal runs are + * unaffected. + * + * The diagnostic machinery is Linux-only (it relies on /proc, real-time + * signals, tgkill and glibc backtrace); on other platforms arm_from_env() is a + * no-op so the tests still build. + */ + +#ifndef TESTS_INCLUDE_TEST_HANG_WATCHDOG_H +#define TESTS_INCLUDE_TEST_HANG_WATCHDOG_H + +#if defined(__linux__) + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/* Optionally provided by another TU (e.g. the MCIPS plugin) to dump extra state + * -- per-vCPU status and sync-window attach flags -- after the backtraces. Weak, + * so it is simply skipped when not linked in. Must read lock-free (the watchdog + * runs while other threads are deadlocked, possibly holding locks). */ +extern "C" void qbox_wd_dump_extra_state() __attribute__((weak)); + +/* Optionally provided by gs_suspend_debug.h: dumps which suspending channels + * (async_event / sync_window) are still attached and where each was attached + * from -- i.e. exactly what is keeping the kernel in async_suspend(). Weak. */ +extern "C" void qbox_wd_dump_suspend_channels() __attribute__((weak)); + +namespace qbox_wd { + +/* Runs in the context of each signalled thread: print its own backtrace. Only + * uses async-signal-safe libc calls (backtrace/backtrace_symbols_fd/write). */ +inline void wd_bt_handler(int /*sig*/) +{ + void* frames[128]; + int n = ::backtrace(frames, 128); + ::backtrace_symbols_fd(frames, n, STDERR_FILENO); + const char nl = '\n'; + ssize_t r = ::write(STDERR_FILENO, &nl, 1); + (void)r; +} + +inline void wd_dump_all_thread_backtraces() +{ + /* Use a real-time signal to avoid colliding with QEMU's SIG_IPI (SIGUSR1). */ + const int WD_SIG = SIGRTMIN + 4; + struct sigaction sa; + std::memset(&sa, 0, sizeof(sa)); + sa.sa_handler = wd_bt_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + sigaction(WD_SIG, &sa, nullptr); + + const pid_t pid = getpid(); + DIR* d = opendir("/proc/self/task"); + if (d) { + struct dirent* e; + while ((e = readdir(d)) != nullptr) { + if (e->d_name[0] == '.') continue; + long tid = strtol(e->d_name, nullptr, 10); + char name[64] = { 0 }; + { + char path[96]; + std::snprintf(path, sizeof(path), "/proc/self/task/%ld/comm", tid); + FILE* f = std::fopen(path, "r"); + if (f) { + if (std::fgets(name, sizeof(name), f)) { + char* nlp = std::strchr(name, '\n'); + if (nlp) *nlp = '\0'; + } + std::fclose(f); + } + } + std::fprintf(stderr, "\n=== WATCHDOG backtrace: tid %ld (%s) ===\n", tid, name); + std::fflush(stderr); + syscall(SYS_tgkill, pid, (int)tid, WD_SIG); + usleep(80000); /* give the handler time to print before the next */ + } + closedir(d); + } + std::fflush(stderr); +} + +inline void wd_dump_state_tail() +{ + if (qbox_wd_dump_extra_state) { + std::fprintf(stderr, "\n######## WATCHDOG extra state ########\n"); + std::fflush(stderr); + qbox_wd_dump_extra_state(); + } + if (qbox_wd_dump_suspend_channels) { + std::fprintf(stderr, "\n######## WATCHDOG suspending channels ########\n"); + std::fflush(stderr); + qbox_wd_dump_suspend_channels(); + } +} + +/* Fatal-signal handler: a real crash (SEGV/ABRT/BUS/...) also gets a full dump so + * intermittent mcips crashes in CI are diagnosable, not just an opaque SEGFAULT. + * Dumps the faulting thread first (most reliable), then all threads + state, then + * restores the default handler and re-raises to preserve the crash/exit status. */ +inline void wd_crash_handler(int sig) +{ + static volatile sig_atomic_t in_handler = 0; + if (in_handler) { + signal(sig, SIG_DFL); + raise(sig); + return; + } /* avoid recursion */ + in_handler = 1; + + std::fprintf(stderr, "\n\n######## QBOX WATCHDOG: caught fatal signal %d ########\n", sig); + std::fflush(stderr); + /* faulting thread's own stack first */ + void* frames[128]; + int n = ::backtrace(frames, 128); + ::backtrace_symbols_fd(frames, n, STDERR_FILENO); + /* then every thread + plugin state (best effort) */ + wd_dump_all_thread_backtraces(); + wd_dump_state_tail(); + std::fprintf(stderr, "\n######## QBOX WATCHDOG: re-raising signal %d ########\n", sig); + std::fflush(stderr); + signal(sig, SIG_DFL); + raise(sig); +} + +inline void install_crash_handlers() +{ + /* alt stack so a SIGSEGV from stack overflow can still be handled */ + static char alt[64 * 1024]; + stack_t ss; + ss.ss_sp = alt; + ss.ss_size = sizeof(alt); + ss.ss_flags = 0; + sigaltstack(&ss, nullptr); + + struct sigaction sa; + std::memset(&sa, 0, sizeof(sa)); + sa.sa_handler = wd_crash_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_ONSTACK; + for (int s : { SIGSEGV, SIGABRT, SIGBUS, SIGFPE, SIGILL }) sigaction(s, &sa, nullptr); +} + +inline void arm(int seconds) +{ + install_crash_handlers(); + std::thread([seconds]() { + std::this_thread::sleep_for(std::chrono::seconds(seconds)); + char cmd[1024] = { 0 }; + { + FILE* f = std::fopen("/proc/self/cmdline", "r"); + if (f) { + size_t n = std::fread(cmd, 1, sizeof(cmd) - 1, f); + for (size_t i = 0; i + 1 < n; i++) + if (cmd[i] == '\0') cmd[i] = ' '; + std::fclose(f); + } + } + std::fprintf(stderr, + "\n\n######## QBOX HANG WATCHDOG fired after %ds ########\n" + "cmd: %s\n" + "######## dumping all thread backtraces ########\n", + seconds, cmd); + std::fflush(stderr); + wd_dump_all_thread_backtraces(); + wd_dump_state_tail(); + std::fprintf(stderr, "\n######## QBOX HANG WATCHDOG: exiting(134) ########\n"); + std::fflush(stderr); + /* _Exit, not abort(), so we don't re-enter the SIGABRT crash handler */ + std::_Exit(134); + }).detach(); +} + +/* True if this process's command line matches a run we want the watchdog to + * cover: an MCIPS run, or the dmi-test-async-inval test (whose QK/COROUTINE + * num-cpu=32 variant times out in CI and we want its stuck-state dumped). */ +inline bool wd_cmdline_should_arm() +{ + FILE* f = std::fopen("/proc/self/cmdline", "r"); + if (!f) return false; + char buf[4096]; + size_t n = std::fread(buf, 1, sizeof(buf) - 1, f); + std::fclose(f); + for (size_t i = 0; i + 1 < n; i++) + if (buf[i] == '\0') buf[i] = ' '; + buf[n < sizeof(buf) ? n : sizeof(buf) - 1] = '\0'; + return std::strstr(buf, "time_sync_strategy=mcips") != nullptr || + std::strstr(buf, "dmi-test-async-inval") != nullptr; +} + +inline void arm_from_env() +{ + const char* s = ::getenv("QBOX_HANG_WATCHDOG_SEC"); + if (!s || !*s) return; + int sec = std::atoi(s); + if (sec <= 0) return; + /* Arm only for the runs under investigation (see wd_cmdline_should_arm) so + * the watchdog never trips on other legitimately-slow tests. */ + if (!wd_cmdline_should_arm()) return; + arm(sec); +} + +} // namespace qbox_wd + +#else // !__linux__ + +namespace qbox_wd { +inline void arm_from_env() {} // no-op on non-Linux +} // namespace qbox_wd + +#endif // __linux__ + +#endif // TESTS_INCLUDE_TEST_HANG_WATCHDOG_H diff --git a/tests/qbox/include/test/test.h b/tests/qbox/include/test/test.h index 64b6834b..1e8344a3 100644 --- a/tests/qbox/include/test/test.h +++ b/tests/qbox/include/test/test.h @@ -20,6 +20,8 @@ #include +#include "test/hang_watchdog.h" + class TestFailureException : public std::runtime_error { protected: @@ -76,6 +78,8 @@ class TestBench : public sc_core::sc_module template int run_testbench(int argc, char* argv[]) { + qbox_wd::arm_from_env(); // debug hang watchdog (no-op unless QBOX_HANG_WATCHDOG_SEC set) + scp::LoggingGuard logging_guard(scp::LogConfig() .fileInfoFrom(sc_core::SC_ERROR) .logAsync(false)