diff --git a/rhi/metal_device.odin b/rhi/metal_device.odin index 219f29c78..11d4ee6ba 100644 --- a/rhi/metal_device.odin +++ b/rhi/metal_device.odin @@ -250,6 +250,7 @@ Device :: struct { failure_text: string, failure_buffer: [FAILURE_TEXT_BYTES]u8, lose_asked: bool, // device_lose + lose_unreported: bool, // device_lose without reported: reported as any loss lose_provoked: bool, refuse: Pipeline_Refusal, presenter: Presenter, diff --git a/rhi/rhi.odin b/rhi/rhi.odin index ab25e67a1..9c51bbdd2 100644 --- a/rhi/rhi.odin +++ b/rhi/rhi.odin @@ -717,13 +717,17 @@ device_failure :: proc(d: ^Device) -> (failure: Device_Failure, text: string) { // Makes the next submission report the device lost, as the API does after a // reset of the GPU, once its work went and completed: what a test of the // reaction to a loss provokes without resetting the GPU of the machine. -device_lose :: proc(d: ^Device) {d.lose_asked = true} +// reported false: the next frame goes as after a reset no call reports (amdvlk +// on Windows after a timeout detection and recovery): its submission and its +// timeline pass, its work never runs. Metal, whose command buffers report +// every reset, and an immediate list report it as without it. +device_lose :: proc(d: ^Device, reported := true) {d.lose_asked, d.lose_unreported = true, !reported} // Whether device_lose asked the submission that just went to report a loss. @(private) device_lose_take :: proc(d: ^Device) -> bool { asked := d.lose_asked - d.lose_asked = false + d.lose_asked, d.lose_unreported = false, false d.lose_provoked ||= asked return asked } diff --git a/rhi/tests/rhi_test.odin b/rhi/tests/rhi_test.odin index 4c524dfc0..44851733b 100644 --- a/rhi/tests/rhi_test.odin +++ b/rhi/tests/rhi_test.odin @@ -114,6 +114,39 @@ rhi_a_lost_immediate_list_reports_the_loss :: proc(t: ^testing.T) { expect_clean(t, d) } +// A reset no call reports (device_lose without reported) is found by the +// frame that reads what its work left, which fails at once with the device +// lost instead of waiting for the GPU. +@(test) +rhi_a_reset_no_call_reports_fails_the_device_at_once :: proc(t: ^testing.T) { + d := test_device(t) + if d == nil {return} + defer rhi.device_destroy(d) + if !rhi.device_info(d).features.timestamps { + fmt.println("skipped: the device has no timestamps") + return + } + buffer := rhi.buffer_create(d, {size = 16, usage = {.Storage}, memory = .GPU, name = "reset"}) + testing.expect(t, buffer != 0, rhi.device_error(d)) + started := time.tick_now() + began := true + for frame in 0 ..= rhi.MAX_FRAMES_IN_FLIGHT { + cmd: ^rhi.Command_List + cmd, began = rhi.frame_begin(d) + if !began {break} + rhi.cmd_fill_buffer(cmd, buffer, 0, 16, u8(frame)) + if frame == 0 {rhi.device_lose(d, reported = false)} + rhi.frame_end(d, cmd) + } + testing.expect(t, !began, "a frame after the reset does not begin") + failure, text := rhi.device_failure(d) + testing.expect_value(t, failure, rhi.Device_Failure.Lost) + testing.expectf(t, strings.has_suffix(text, "(provoked by device_lose)"), "the cause says it was provoked: %s", text) + testing.expect(t, time.tick_since(started) < time.Second, "nothing waits for the GPU") + rhi.buffer_destroy(d, buffer) + expect_clean(t, d) +} + @(test) rhi_buffer_round_trip_and_compute :: proc(t: ^testing.T) { d := test_device(t) diff --git a/rhi/vulkan_device.odin b/rhi/vulkan_device.odin index d67b347ff..fe59e84fb 100644 --- a/rhi/vulkan_device.odin +++ b/rhi/vulkan_device.odin @@ -270,6 +270,7 @@ Device :: struct { memory_budget: bool, // VK_EXT_memory_budget: the driver says what memory the process holds and may hold waited_out: bool, // a wait passed GPU_WAIT_LIMIT_NS: the GPU may still run that work lose_asked: bool, // device_lose + lose_unreported: bool, // device_lose without reported lose_provoked: bool, refuse: Pipeline_Refusal, presenter: Presenter, @@ -1176,8 +1177,18 @@ vk_collect_timestamps :: proc(d: ^Device, f: ^Vk_Frame) { f.submitted = false if f.queries == 0 || count < 2 {return} values: [MAX_TIMESTAMPS]u64 - // The slot's submission already completed, so WAIT returns immediately. - result := d.dvk.GetQueryPoolResults(d.handle, f.queries, 0, count, size_of(values), &values, size_of(u64), {._64, .WAIT}) + // The timeline passed the slot's submission, so its timestamps are written + // unless the GPU was reset under it. A driver may report such a reset + // nowhere and never write them (amdvlk on Windows after a timeout + // detection and recovery): a WAIT for them would not return. + result := d.dvk.GetQueryPoolResults(d.handle, f.queries, 0, count, size_of(values), &values, size_of(u64), {._64}) + if result == .NOT_READY { + d.timing_measured = false + stopped_buffer: [256]u8 + stopped := vk_stopped(d, stopped_buffer[:]) + device_fail(d, .Lost, "frame %d ended without the work it was given: the GPU was reset under it%s%s", f.number, "; the GPU stopped in " if stopped != "" else "", stopped) + return + } if result != .SUCCESS {d.timing_measured = false; vk_lost(d, result, "vkGetQueryPoolResults"); return} gpu, now := values[count - 1], time.duration_nanoseconds(time.tick_diff({}, time.tick_now())) if d.calibrate != nil { @@ -1220,6 +1231,7 @@ frame_begin :: proc(d: ^Device, present := true) -> (cmd: ^Command_List, ok: boo // Before the deletions: the copy of this slot stops naming what goes. vk_table_take(d, table_copy(false, d.slot)) vk_collect_timestamps(d, f) + if d.failure != .None {return nil, false} vk_flush_deletions(d, f) // What waited for the uploads sent since the last frame waits for this one, which follows them. if len(d.unframed) > 0 { @@ -1306,9 +1318,16 @@ frame_end :: proc(d: ^Device, cmd: ^Command_List, generated := false) -> bool { } submit.signalSemaphoreInfoCount = u32(signal_count) submit.pSignalSemaphoreInfos = raw_data(signals[:]) + // A reset no call reports: the semaphores pass, the list and its timestamps do not run. + unreported := d.lose_asked && d.lose_unreported + if unreported { + d.lose_asked, d.lose_unreported, d.lose_provoked = false, false, true + if f.queries != 0 {d.dvk.ResetQueryPool(d.handle, f.queries, 0, MAX_TIMESTAMPS)} + submit.commandBufferInfoCount = 0 + } sync.mutex_lock(&d.queue_lock) submitted := d.dvk.QueueSubmit2(d.queue, 1, &submit, 0) - f.marks.sent = submitted == .SUCCESS + f.marks.sent = submitted == .SUCCESS && !unreported ok := vk_check(d, submitted, "vkQueueSubmit2") f.signal_value = d.timeline_value f.number, f.submitted = d.frame_number, submitted == .SUCCESS