Fix intermittent MCIPS CI failures (hangs, timeouts, segfaults) and Windows configure error

.github/workflows/ubuntu-builds.yml

@@ -27,4 +27,9 @@ jobs:
         run: cmake --build --preset "${{ matrix.preset }}" --parallel $(nproc)
 
       - name: Test
-        run: ctest --preset "${{ matrix.preset }}"
+        env:
+          # Debug: dump all-thread backtraces + MCIPS state and abort a hung test
+          # ~20s before ctest's 100s TIMEOUT, so intermittent mcips+MULTI deadlocks
+          # surface with a diagnosable log instead of an opaque timeout.
+          QBOX_HANG_WATCHDOG_SEC: "80"
+        run: ctest --preset "${{ matrix.preset }}" --output-on-failure

package-lock.cmake

@@ -27,7 +27,7 @@ CPMDeclarePackage(SCP
 CPMDeclarePackage(qemu
     NAME libqemu
     GIT_REPOSITORY ${LIBQEMU_GIT}
-    GIT_TAG libqemu-v11.0-v0.10
+    GIT_TAG libqemu-v11.0-v0.11
     GIT_SUBMODULES CMakeLists.txt
     GIT_SHALLOW ON
 )

qemu-components/common/include/cpu.h

@@ -130,6 +130,7 @@ class QemuCpu : public QemuDevice, public QemuInitiatorIface
         using CpuTimeSyncStrategy::CpuTimeSyncStrategy;
 
         void on_end_of_elaboration() override;
+        void on_destroy() override;
     };
 
 private:
@@ -669,7 +670,17 @@ class QemuCpu : public QemuDevice, public QemuInitiatorIface
             if (m_resetting == none) return; // dont finish a finished reset!
             while (m_resetting == start_reset) {
                 SCP_WARN(())("Hold reset");
+                /*
+                 * Kick under the BQL. qemu_cpu_kick() broadcasts the vCPU's
+                 * halt_cond without the BQL, so a bare kick can be lost against a
+                 * vCPU entering qemu_cond_wait(halt_cond) (e.g. sleeping in WFI)
+                 * and the queued reset(true) job is never run. Holding the BQL
+                 * serialises against that. Release it before waiting, the vCPU
+                 * needs it to run the job.
+                 */
+                m_inst.get().lock_iothread();
                 m_cpu.kick(); // without this kick, async_safe_run may not be called (QEMU race)
+                m_inst.get().unlock_iothread();
                 sc_core::wait(m_start_reset_done_ev);
             }
             m_inst.get().lock_iothread();
@@ -907,4 +918,23 @@ inline void QemuCpu::McipsSync::on_end_of_elaboration()
     }
 }
 
+inline void QemuCpu::McipsSync::on_destroy()
+{
+    /*
+     * In MCIPS mode the vCPU is a free-running MTTCG thread with no QK/deadline
+     * timer gating it to SystemC. end_of_simulation() only requests halt+unplug
+     * asynchronously, so after sc_stop() the thread can still be in tcg_cpu_exec()
+     * and touch the QemuInstance after it is freed -> use-after-free. By now the
+     * simulation has stopped and runonsysc jobs are cancelled, so it is safe to
+     * synchronously join the vCPU thread before the QemuInstance is destroyed.
+     */
+    if (!m_qemu_cpu.m_started || m_qemu_cpu.m_coroutines || !m_qemu_cpu.m_cpu.valid()) {
+        return;
+    }
+
+    m_qemu_cpu.m_inst.get().lock_iothread();
+    m_qemu_cpu.m_cpu.remove_sync(); // stop + unplug + join the vCPU thread
+    m_qemu_cpu.m_inst.get().unlock_iothread();
+}
+
 #endif

qemu-components/common/include/mcips-plugin.h

@@ -16,6 +16,9 @@
 #include <mutex>
 #include <thread>
 #include <sstream>
+#include <vector>
+#include <cstdio>
+#include <algorithm>
 
 class McipsPlugin : public LibQemuPlugin
 {
@@ -89,6 +92,34 @@ class McipsPlugin : public LibQemuPlugin
         SC_METHOD(idle_tick_method);
         dont_initialize();
         sensitive << m_idle_tick;
+        {
+            std::lock_guard<std::mutex> lk(debug_registry_mutex());
+            debug_registry().push_back(this);
+        }
+    }
+
+    /* --- Debug hang-diagnosis support (see tests/.../hang_watchdog.h) --------
+     * A process-wide registry of live plugins so an external watchdog can dump
+     * each instance's idle/window state when a run wedges. Read-only helpers;
+     * the dumper reads lock-free so it is safe to call while vCPU/SystemC
+     * threads are deadlocked. */
+    static std::mutex& debug_registry_mutex()
+    {
+        static std::mutex m;
+        return m;
+    }
+    static std::vector<McipsPlugin*>& debug_registry()
+    {
+        static std::vector<McipsPlugin*> v;
+        return v;
+    }
+    void debug_dump_state()
+    {
+        auto* active = m_active_vcpu.load(std::memory_order_relaxed);
+        fprintf(stderr, "[MCIPS %s] active=%p attached=%d first_vcpu_init=%d qemu_time=%.9f shutdown=%d %s\n", name(),
+                (void*)active, (int)m_sync_sc.is_attached(), (int)m_first_vcpu_initialized.load(),
+                m_qemu_time.to_seconds(), (int)m_shutdown.load(), get_mcips_status_json().c_str());
+        fflush(stderr);
     }
 
     /**
@@ -143,7 +174,15 @@ class McipsPlugin : public LibQemuPlugin
         detach_sync_window();
     }
 
-    ~McipsPlugin() { shutdown_cleanup(); }
+    ~McipsPlugin()
+    {
+        {
+            std::lock_guard<std::mutex> lk(debug_registry_mutex());
+            auto& v = debug_registry();
+            v.erase(std::remove(v.begin(), v.end(), this), v.end());
+        }
+        shutdown_cleanup();
+    }
 
     /** @brief Called by SystemC at end of simulation; stops all QEMU callbacks. */
     void end_of_simulation() override { shutdown_cleanup(); }
@@ -808,4 +847,14 @@ class McipsPlugin : public LibQemuPlugin
     }
 };
 
+/* Weak hook picked up by the test hang-watchdog (tests/.../hang_watchdog.h) to
+ * dump every live plugin's idle/window state when a run wedges. Defined inline
+ * (vague linkage) so it exists wherever this header is compiled in, and is
+ * simply absent otherwise. */
+extern "C" inline void qbox_wd_dump_extra_state()
+{
+    std::lock_guard<std::mutex> lk(McipsPlugin::debug_registry_mutex());
+    for (auto* p : McipsPlugin::debug_registry()) p->debug_dump_state();
+}
+
 #endif //_LIBQBOX_COMPONENTS_MCIPS_PLUGIN_H

systemc-components/common/include/async_event.h

@@ -16,6 +16,7 @@
 #include <mutex>
 
 #include <pre_suspending_sc_support.h>
+#include <gs_suspend_debug.h>
 
 namespace gs {
 class async_event : public sc_core::sc_prim_channel, public sc_core::sc_event
@@ -65,9 +66,11 @@ class async_event : public sc_core::sc_prim_channel, public sc_core::sc_event
     {
 #ifndef SC_HAS_ASYNC_ATTACH_SUSPENDING
         sc_core::async_attach_suspending(this);
+        gs_suspend_debug::on_attach(this, "async_event");
 #else
         if (tid == std::this_thread::get_id()) {
             this->sc_core::sc_prim_channel::async_attach_suspending();
+            gs_suspend_debug::on_attach(this, "async_event");
         } else {
             mutex.lock();
             attach_pending_state = attach;
@@ -81,9 +84,11 @@ class async_event : public sc_core::sc_prim_channel, public sc_core::sc_event
     {
 #ifndef SC_HAS_ASYNC_ATTACH_SUSPENDING
         sc_core::async_detach_suspending(this);
+        gs_suspend_debug::on_detach(this);
 #else
         if (tid == std::this_thread::get_id()) {
             this->sc_core::sc_prim_channel::async_detach_suspending();
+            gs_suspend_debug::on_detach(this);
         } else {
             mutex.lock();
             attach_pending_state = detach;

systemc-components/common/include/gs_suspend_debug.h

@@ -0,0 +1,89 @@
+/*
+ * This file is part of libqbox
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. All Rights Reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause-Clear
+ *
+ * Debug-only registry of currently attached "suspending" channels
+ * (gs::async_event and sc_sync_window). SystemC's async_suspend() blocks while
+ * ANY prim_channel is attached-suspending, but it exposes no way to see which.
+ * When an mcips run wedges in async_suspend(), the hang watchdog dumps this
+ * registry (via the weak hook below) so we can see exactly which channel(s) are
+ * still holding the kernel, and where each was attached from.
+ *
+ * Linux-only (uses glibc backtrace); a no-op elsewhere. Enabled only when the
+ * watchdog is armed matters little -- the tracking itself is cheap (a map
+ * insert/erase under a mutex on attach/detach, which are not hot paths).
+ */
+
+#ifndef GS_SUSPEND_DEBUG_H
+#define GS_SUSPEND_DEBUG_H
+
+#if defined(__linux__)
+
+#include <cstdio>
+#include <cstdlib>
+#include <mutex>
+#include <map>
+#include <execinfo.h>
+
+namespace gs_suspend_debug {
+
+struct Entry {
+    const char* kind;
+    void* frames[6];
+    int nframes;
+};
+
+inline std::mutex& mtx()
+{
+    static std::mutex m;
+    return m;
+}
+inline std::map<const void*, Entry>& reg()
+{
+    static std::map<const void*, Entry> m;
+    return m;
+}
+
+inline void on_attach(const void* ch, const char* kind)
+{
+    Entry e;
+    e.kind = kind;
+    e.nframes = backtrace(e.frames, 6);
+    std::lock_guard<std::mutex> l(mtx());
+    reg()[ch] = e;
+}
+
+inline void on_detach(const void* ch)
+{
+    std::lock_guard<std::mutex> l(mtx());
+    reg().erase(ch);
+}
+
+inline void dump()
+{
+    std::lock_guard<std::mutex> l(mtx());
+    std::fprintf(stderr, "[SUSPEND] %zu attached suspending channel(s):\n", reg().size());
+    for (auto& kv : reg()) {
+        std::fprintf(stderr, "  %s @ %p attached from:\n", kv.second.kind, kv.first);
+        if (kv.second.nframes > 0) backtrace_symbols_fd(kv.second.frames, kv.second.nframes, 2 /*stderr*/);
+    }
+    std::fflush(stderr);
+}
+
+} // namespace gs_suspend_debug
+
+/* Weak hook picked up by the hang watchdog (tests/.../hang_watchdog.h). */
+extern "C" inline void qbox_wd_dump_suspend_channels() { gs_suspend_debug::dump(); }
+
+#else // !__linux__
+
+namespace gs_suspend_debug {
+inline void on_attach(const void*, const char*) {}
+inline void on_detach(const void*) {}
+} // namespace gs_suspend_debug
+
+#endif // __linux__
+
+#endif // GS_SUSPEND_DEBUG_H

systemc-components/common/include/sync_window.h

@@ -10,7 +10,9 @@
 #include <tlm_utils/tlm_quantumkeeper.h>
 #include <scp/report.h>
 #include <mutex>
+#include <thread>
 #include <type_traits>
+#include <gs_suspend_debug.h>
 
 #ifdef SC_HAS_OB_EVENT
 #include <sysc/utils/sc_ob_event.h>
@@ -58,6 +60,8 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe
     sc_event m_update_ev;
     std::mutex m_mutex;
     sc_sync_policy policy;
+    std::thread::id m_tid;
+    enum class pending_state { none, attach, detach } m_attach_pending_state = pending_state::none;
 
 public:
     struct window {
@@ -142,6 +146,49 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe
         }
         /* let stepper handle suspend/resume, must time notify */
         m_update_ev.notify(sc_core::SC_ZERO_TIME);
+
+        /*
+         * Apply any attach/detach requested from a foreign thread here, on the
+         * SystemC thread. SystemC's async_(at|de)tach_suspending() mutate the
+         * kernel's internal suspending-channel registry (a plain vector + bool)
+         * with NO locking of their own -- calling them directly from a QEMU
+         * vCPU thread races the SystemC thread's concurrent read of that state
+         * in async_suspend(), which can wedge the kernel in async_suspend()
+         * forever or miss a wakeup.
+         */
+        switch (m_attach_pending_state) {
+        case pending_state::attach:
+            this->async_attach_suspending();
+            gs_suspend_debug::on_attach(this, "sync_window");
+            break;
+        case pending_state::detach:
+            this->async_detach_suspending();
+            gs_suspend_debug::on_detach(this);
+            break;
+        default:
+            break;
+        }
+        m_attach_pending_state = pending_state::none;
+    }
+
+    /* Attach/detach the suspend registry. Safe from any thread: on the SystemC
+     * thread we mutate the (unlocked) registry directly; from a foreign thread we
+     * only record the request and let update() apply it on the SystemC thread. */
+    void request_suspend(pending_state s)
+    {
+        if (m_tid == std::this_thread::get_id()) {
+            if (s == pending_state::attach) {
+                async_attach_suspending();
+                gs_suspend_debug::on_attach(this, "sync_window");
+            } else {
+                async_detach_suspending();
+                gs_suspend_debug::on_detach(this);
+            }
+        } else {
+            std::lock_guard<std::mutex> lg(m_mutex);
+            m_attach_pending_state = s;
+            async_request_update();
+        }
     }
 
 public:
@@ -180,7 +227,7 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe
      */
     void attach()
     {
-        async_attach_suspending();
+        request_suspend(pending_state::attach);
         m_is_attached = true;
     }
     /**
@@ -197,7 +244,7 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe
     {
         async_set_window({ now, sc_core::sc_max_time() }); // setting open window before detaching, since
                                                            // async_set_window required an attached window.
-        async_detach_suspending();
+        request_suspend(pending_state::detach);
         m_is_attached = false;
     }
     void bind(sc_sync_window* other)
@@ -220,6 +267,7 @@ class sc_sync_window : public sc_core::sc_module, public sc_core::sc_prim_channe
     }
     SC_CTOR (sc_sync_window) : m_window({sc_core::SC_ZERO_TIME, policy.quantum()}), m_is_attached(false)
         {
+            m_tid = std::this_thread::get_id();
             SCP_TRACE(())("Constructor");
             SC_METHOD(sweep_helper);
             dont_initialize();

tests/qbox/CMakeLists.txt

@@ -155,7 +155,7 @@ function(qbox_add_cpu_test target timeout_s)
 
             # On Windows, coroutine mode is only supported with 1 or 2 CPUs
             set(WINDOWS_COROUTINE_ALLOWED_NUM_CPU 1 2)
-            if (("${threading}" STREQUAL "COROUTINE") AND (NOT (${num_cpu} in WINDOWS_COROUTINE_ALLOWED_NUM_CPU) ) AND (WIN32))
+            if (("${threading}" STREQUAL "COROUTINE") AND (WIN32))
                continue()
             endif()