From b0ee43837b292f09cf89b7271b09fabbcadef850 Mon Sep 17 00:00:00 2001 From: 5hif7y <44300833+5hif7y@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:29:34 -0300 Subject: [PATCH 1/2] rhi: a GPU reset that skips a frame's timestamps fails the device After a Windows timeout detection and recovery, amdvlk reports the reset neither at submission nor in vkWaitSemaphores: the timeline reaches the value but the frame's timestamp queries are never written. vk_collect_timestamps read them with VK_QUERY_RESULT_WAIT_BIT, which has no timeout, so frame_begin spun in the driver forever. A GPU test that hit this held the shared device, and every later GPU test of the process waited for it until `mirai test` killed the executable. The results are now read without WAIT. NOT_READY after the timeline passed means the submission ended without its work: the device is marked Lost with the place the GPU stopped, and frame_begin returns false. Verified with a forced reset on an RX 580 (one ~7 s Ddgi_Trace dispatch beside another GPU process). The test fails in seconds with "frame ended without the work it was given: the GPU was reset under it; the GPU stopped in ddgi, with the pipeline Ddgi_Trace", and the executable ends. Co-Authored-By: Claude Opus 5.5 --- rhi/vulkan_device.odin | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/rhi/vulkan_device.odin b/rhi/vulkan_device.odin index d67b347ff..dda476fe8 100644 --- a/rhi/vulkan_device.odin +++ b/rhi/vulkan_device.odin @@ -1176,8 +1176,18 @@ vk_collect_timestamps :: proc(d: ^Device, f: ^Vk_Frame) { f.submitted = false if f.queries == 0 || count < 2 {return} values: [MAX_TIMESTAMPS]u64 - // The slot's submission already completed, so WAIT returns immediately. - result := d.dvk.GetQueryPoolResults(d.handle, f.queries, 0, count, size_of(values), &values, size_of(u64), {._64, .WAIT}) + // The timeline passed the slot's submission, so its timestamps are written + // unless the GPU was reset under it. A driver may report such a reset + // nowhere and never write them (amdvlk on Windows after a timeout + // detection and recovery): a WAIT for them would not return. + result := d.dvk.GetQueryPoolResults(d.handle, f.queries, 0, count, size_of(values), &values, size_of(u64), {._64}) + if result == .NOT_READY { + d.timing_measured = false + stopped_buffer: [256]u8 + stopped := vk_stopped(d, stopped_buffer[:]) + device_fail(d, .Lost, "frame %d ended without the work it was given: the GPU was reset under it%s%s", f.number, "; the GPU stopped in " if stopped != "" else "", stopped) + return + } if result != .SUCCESS {d.timing_measured = false; vk_lost(d, result, "vkGetQueryPoolResults"); return} gpu, now := values[count - 1], time.duration_nanoseconds(time.tick_diff({}, time.tick_now())) if d.calibrate != nil { @@ -1220,6 +1230,7 @@ frame_begin :: proc(d: ^Device, present := true) -> (cmd: ^Command_List, ok: boo // Before the deletions: the copy of this slot stops naming what goes. vk_table_take(d, table_copy(false, d.slot)) vk_collect_timestamps(d, f) + if d.failure != .None {return nil, false} vk_flush_deletions(d, f) // What waited for the uploads sent since the last frame waits for this one, which follows them. if len(d.unframed) > 0 { From 156724f8a044517d0da7158f7b7b2dd005c28e2e Mon Sep 17 00:00:00 2001 From: 5hif7y <44300833+5hif7y@users.noreply.github.com> Date: Sun, 15 Nov 2026 11:05:33 -0300 Subject: [PATCH 2/2] rhi: device_lose provokes a reset that no call reports `device_lose(d, reported = false)` makes the next frame go as after a reset that amdvlk reports nowhere: its submission passes its semaphores and its timeline, its command list does not run and its timestamp queries are reset from the host. Metal, whose command buffers report every reset, and an immediate list report it as device_lose does. rhi_a_reset_no_call_reports_fails_the_device_at_once covers the previous commit: with WAIT back in vk_collect_timestamps it hangs until `mirai test` kills it; with the fix the next frame on that slot fails at once with the device Lost. Co-Authored-By: Claude Opus 5.5 --- rhi/metal_device.odin | 1 + rhi/rhi.odin | 8 ++++++-- rhi/tests/rhi_test.odin | 33 +++++++++++++++++++++++++++++++++ rhi/vulkan_device.odin | 10 +++++++++- 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/rhi/metal_device.odin b/rhi/metal_device.odin index 219f29c78..11d4ee6ba 100644 --- a/rhi/metal_device.odin +++ b/rhi/metal_device.odin @@ -250,6 +250,7 @@ Device :: struct { failure_text: string, failure_buffer: [FAILURE_TEXT_BYTES]u8, lose_asked: bool, // device_lose + lose_unreported: bool, // device_lose without reported: reported as any loss lose_provoked: bool, refuse: Pipeline_Refusal, presenter: Presenter, diff --git a/rhi/rhi.odin b/rhi/rhi.odin index ab25e67a1..9c51bbdd2 100644 --- a/rhi/rhi.odin +++ b/rhi/rhi.odin @@ -717,13 +717,17 @@ device_failure :: proc(d: ^Device) -> (failure: Device_Failure, text: string) { // Makes the next submission report the device lost, as the API does after a // reset of the GPU, once its work went and completed: what a test of the // reaction to a loss provokes without resetting the GPU of the machine. -device_lose :: proc(d: ^Device) {d.lose_asked = true} +// reported false: the next frame goes as after a reset no call reports (amdvlk +// on Windows after a timeout detection and recovery): its submission and its +// timeline pass, its work never runs. Metal, whose command buffers report +// every reset, and an immediate list report it as without it. +device_lose :: proc(d: ^Device, reported := true) {d.lose_asked, d.lose_unreported = true, !reported} // Whether device_lose asked the submission that just went to report a loss. @(private) device_lose_take :: proc(d: ^Device) -> bool { asked := d.lose_asked - d.lose_asked = false + d.lose_asked, d.lose_unreported = false, false d.lose_provoked ||= asked return asked } diff --git a/rhi/tests/rhi_test.odin b/rhi/tests/rhi_test.odin index 4c524dfc0..44851733b 100644 --- a/rhi/tests/rhi_test.odin +++ b/rhi/tests/rhi_test.odin @@ -114,6 +114,39 @@ rhi_a_lost_immediate_list_reports_the_loss :: proc(t: ^testing.T) { expect_clean(t, d) } +// A reset no call reports (device_lose without reported) is found by the +// frame that reads what its work left, which fails at once with the device +// lost instead of waiting for the GPU. +@(test) +rhi_a_reset_no_call_reports_fails_the_device_at_once :: proc(t: ^testing.T) { + d := test_device(t) + if d == nil {return} + defer rhi.device_destroy(d) + if !rhi.device_info(d).features.timestamps { + fmt.println("skipped: the device has no timestamps") + return + } + buffer := rhi.buffer_create(d, {size = 16, usage = {.Storage}, memory = .GPU, name = "reset"}) + testing.expect(t, buffer != 0, rhi.device_error(d)) + started := time.tick_now() + began := true + for frame in 0 ..= rhi.MAX_FRAMES_IN_FLIGHT { + cmd: ^rhi.Command_List + cmd, began = rhi.frame_begin(d) + if !began {break} + rhi.cmd_fill_buffer(cmd, buffer, 0, 16, u8(frame)) + if frame == 0 {rhi.device_lose(d, reported = false)} + rhi.frame_end(d, cmd) + } + testing.expect(t, !began, "a frame after the reset does not begin") + failure, text := rhi.device_failure(d) + testing.expect_value(t, failure, rhi.Device_Failure.Lost) + testing.expectf(t, strings.has_suffix(text, "(provoked by device_lose)"), "the cause says it was provoked: %s", text) + testing.expect(t, time.tick_since(started) < time.Second, "nothing waits for the GPU") + rhi.buffer_destroy(d, buffer) + expect_clean(t, d) +} + @(test) rhi_buffer_round_trip_and_compute :: proc(t: ^testing.T) { d := test_device(t) diff --git a/rhi/vulkan_device.odin b/rhi/vulkan_device.odin index dda476fe8..fe59e84fb 100644 --- a/rhi/vulkan_device.odin +++ b/rhi/vulkan_device.odin @@ -270,6 +270,7 @@ Device :: struct { memory_budget: bool, // VK_EXT_memory_budget: the driver says what memory the process holds and may hold waited_out: bool, // a wait passed GPU_WAIT_LIMIT_NS: the GPU may still run that work lose_asked: bool, // device_lose + lose_unreported: bool, // device_lose without reported lose_provoked: bool, refuse: Pipeline_Refusal, presenter: Presenter, @@ -1317,9 +1318,16 @@ frame_end :: proc(d: ^Device, cmd: ^Command_List, generated := false) -> bool { } submit.signalSemaphoreInfoCount = u32(signal_count) submit.pSignalSemaphoreInfos = raw_data(signals[:]) + // A reset no call reports: the semaphores pass, the list and its timestamps do not run. + unreported := d.lose_asked && d.lose_unreported + if unreported { + d.lose_asked, d.lose_unreported, d.lose_provoked = false, false, true + if f.queries != 0 {d.dvk.ResetQueryPool(d.handle, f.queries, 0, MAX_TIMESTAMPS)} + submit.commandBufferInfoCount = 0 + } sync.mutex_lock(&d.queue_lock) submitted := d.dvk.QueueSubmit2(d.queue, 1, &submit, 0) - f.marks.sent = submitted == .SUCCESS + f.marks.sent = submitted == .SUCCESS && !unreported ok := vk_check(d, submitted, "vkQueueSubmit2") f.signal_value = d.timeline_value f.number, f.submitted = d.frame_number, submitted == .SUCCESS