From 183d2a5edc4d68fadb166776b8531a2b4becfb7a Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Wed, 30 Sep 2026 23:54:10 +0200 Subject: [PATCH 1/2] perf(windows): repeat the last frame instead of reading back an unchanged screen The video writer read the full GPU frame back on every tick, even when WGC delivered nothing new: a CopyResource, a Map that waits on the GPU, and the BGRA to NV12 conversion, for a picture it already had. On a static screen, most of a demo, that was every tick. When no new WGC frame arrived and no new camera frame is drawn into the picture, the encoder now hands over a new sample on the last buffer. Measured at 1080p: 5.0 ms per readback, under 0.001 ms per repeat. The DXGI path and the legacy callback path read back as before. Fixes #925 --- electron/native/wgc-capture/src/main.cpp | 13 ++- .../native/wgc-capture/src/mf_encoder.cpp | 20 ++++ electron/native/wgc-capture/src/mf_encoder.h | 6 ++ .../wgc-capture/src/mf_encoder_color_test.cpp | 92 +++++++++++++++++++ 4 files changed, 130 insertions(+), 1 deletion(-) diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index a6bba7c76..1bad39699 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -1101,6 +1101,10 @@ int wmain(int argc, wchar_t* argv[]) { Microsoft::WRL::ComPtr webcamSample; bool hasVideoSample = false; bool hasWebcamSample = false; + // Whether the picture this tick encodes differs from the last one: + // a new WGC frame, or a new camera frame drawn into it. The legacy + // callback path cannot tell, so it always reads back. + bool pictureChanged = legacyFrameCallback; std::unique_lock legacyLock; { @@ -1139,6 +1143,7 @@ int wmain(int argc, wchar_t* argv[]) { ID3D11Texture2D* wgcTexture = nullptr; int64_t wgcTimestampHns = 0; const bool gotFrame = session.tryGetNextFrame(&wgcTexture, &wgcTimestampHns); + pictureChanged = gotFrame; if (gotFrame) { if (!latestFrameTexture) { D3D11_TEXTURE2D_DESC desc{}; @@ -1181,6 +1186,7 @@ int wmain(int argc, wchar_t* argv[]) { latestWebcamHeight = candidateWebcamFrame.height; latestWebcamSequence = candidateWebcamFrame.sequence; hasVisibleWebcamFrame = true; + pictureChanged = pictureChanged || !writeSeparateWebcam; } } const BgraFrameView webcamFrame{ @@ -1274,7 +1280,12 @@ int wmain(int argc, wchar_t* argv[]) { // this struct's request: it falls back to the CPU path on // its own when the GPU path does not fit the machine. bool captured = false; - if (usesDxgiInput) { + if (!usesDxgiInput && !pictureChanged && + encoder.repeatLastVideoSample(frameTimestampHns, videoSample)) { + // Nothing new to read back (#925): the static screen, + // most of a demo, costs no GPU copy and no conversion. + captured = true; + } else if (usesDxgiInput) { captured = encoder.captureDxgiSample( latestFrameTexture.Get(), frameTimestampHns, diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index a1af5af79..9eac8c4b4 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -1761,6 +1761,26 @@ bool MFEncoder::captureVideoSample( sample->SetSampleTime(sampleTime); sample->SetSampleDuration(sampleDuration); + lastVideoBuffer_ = buffer; + outSample = sample; + return true; +} + +bool MFEncoder::repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr& outSample) { + outSample.Reset(); + if (!lastVideoBuffer_) { + return false; + } + const int64_t sampleDuration = 10'000'000LL / fps_; + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample(repeat)")) { + return false; + } + // Shared, not copied: a buffer is never written again once its sample is + // built, so two samples can hand the encoder the same bytes. + sample->AddBuffer(lastVideoBuffer_.Get()); + sample->SetSampleTime(nextSampleTime(timestampHns, sampleDuration)); + sample->SetSampleDuration(sampleDuration); outSample = sample; return true; } diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h index 468ca23e3..455624135 100644 --- a/electron/native/wgc-capture/src/mf_encoder.h +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -138,6 +138,11 @@ class MFEncoder { int64_t timestampHns, const BgraFrameView* webcamFrame, Microsoft::WRL::ComPtr& outSample); + // The last frame captureVideoSample read back, again, at a new time: for a + // tick on which WGC delivered nothing new. No GPU copy, no Map, no + // conversion -- the readback a static screen used to pay on every tick + // (getopenscreen/openscreen#925). False before any frame was captured. + bool repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr& outSample); bool captureDxgiSample( ID3D11Texture2D* texture, int64_t timestampHns, @@ -257,6 +262,7 @@ class MFEncoder { // The BGRA frame when it has to be drawn on or rescaled before the NV12 // conversion; reused so a frame does not allocate one. std::vector bgraScratch_; + Microsoft::WRL::ComPtr lastVideoBuffer_; int width_ = 0; int height_ = 0; int fps_ = 60; diff --git a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp index 1faf61ff0..dad217a1c 100644 --- a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp +++ b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp @@ -267,6 +267,97 @@ void timeFullHd(ID3D11Device* device, ID3D11DeviceContext* context) { << captureMs / frames << " ms, submit " << submitMs / frames << " ms per frame" << std::endl; } +// A tick with no new WGC frame repeats the last buffer instead of reading the +// texture back again (getopenscreen/openscreen#925). The file must still hold +// one frame per tick, each the captured picture; and the repeat must cost next +// to nothing next to a readback, or it is not worth having. +void checkRepeatedFrames(ID3D11Device* device, ID3D11DeviceContext* context) { + const auto makeTexture = [&](int width, int height, Microsoft::WRL::ComPtr& texture) { + std::vector bgra(static_cast(width) * height * 4); + for (int y = 0; y < height; y += 1) { + for (int x = 0; x < width; x += 1) { + BYTE* pixel = &bgra[(static_cast(y) * width + x) * 4]; + pixel[2] = 255; // solid red + pixel[3] = 255; + } + } + D3D11_TEXTURE2D_DESC desc{}; + desc.Width = width; + desc.Height = height; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + D3D11_SUBRESOURCE_DATA initial{bgra.data(), static_cast(width * 4), 0}; + return SUCCEEDED(device->CreateTexture2D(&desc, &initial, &texture)); + }; + char tempDir[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tempDir); + + { + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat.mp4"; + const std::wstring widePath(path.begin(), path.end()); + Microsoft::WRL::ComPtr texture; + MFEncoder encoder; + if (!makeTexture(kWidth, kHeight, texture) || + !encoder.initialize(widePath, kWidth, kHeight, 30, 2'000'000, device, context, nullptr, {})) { + expect("repeat-setup", false, "texture or encoder"); + return; + } + Microsoft::WRL::ComPtr sample; + bool wrote = !encoder.repeatLastVideoSample(0, sample) && // nothing to repeat yet + encoder.captureVideoSample(texture.Get(), 0, nullptr, sample) && encoder.submitVideoSample(sample.Get()); + for (int i = 1; i < kFrames && wrote; i += 1) { + wrote = encoder.repeatLastVideoSample(static_cast(i) * 333'333, sample) && + encoder.submitVideoSample(sample.Get()); + } + expect("repeat-writes", wrote && encoder.finalize(), "write or finalize failed"); + const std::string frames = run( + "ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames " + "-of default=nw=1:nk=1 \"" + path + "\""); + const std::string yuv = run( + "ffmpeg -v error -sseof -0.2 -i \"" + path + "\" -frames:v 1 -f rawvideo -pix_fmt yuv420p -"); + const int luma = yuv.empty() ? -1 : static_cast(yuv[static_cast(kHeight / 2) * kWidth + kWidth / 2]); + std::cout << "REPEAT_RAW frames=" << std::atoi(frames.c_str()) << " last-frame Y=" << luma << std::endl; + expect("repeat-keeps-one-frame-per-tick", std::atoi(frames.c_str()) == kFrames, frames); + expect("repeat-keeps-the-picture", std::abs(luma - 63) <= 3, "Y=" + std::to_string(luma)); + DeleteFileA(path.c_str()); + } + + { + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat-timing.mp4"; + const std::wstring widePath(path.begin(), path.end()); + Microsoft::WRL::ComPtr texture; + MFEncoder encoder; + if (!makeTexture(1920, 1080, texture) || + !encoder.initialize(widePath, 1920, 1080, 60, 18'000'000, device, context, nullptr, {})) { + return; + } + double readbackMs = 0.0; + double repeatMs = 0.0; + constexpr int rounds = 60; + for (int i = 0; i < rounds; i += 1) { + Microsoft::WRL::ComPtr sample; + const auto start = std::chrono::steady_clock::now(); + encoder.captureVideoSample(texture.Get(), static_cast(2 * i) * 166'667, nullptr, sample); + const auto read = std::chrono::steady_clock::now(); + encoder.submitVideoSample(sample.Get()); + const auto repeatStart = std::chrono::steady_clock::now(); + encoder.repeatLastVideoSample(static_cast(2 * i + 1) * 166'667, sample); + const auto repeated = std::chrono::steady_clock::now(); + encoder.submitVideoSample(sample.Get()); + readbackMs += std::chrono::duration(read - start).count(); + repeatMs += std::chrono::duration(repeated - repeatStart).count(); + } + encoder.finalize(); + DeleteFileA(path.c_str()); + std::cout << "REPEAT_RAW 1080p readback " << readbackMs / rounds << " ms, repeat " << repeatMs / rounds + << " ms per frame" << std::endl; + } +} + } // namespace int main() { @@ -294,6 +385,7 @@ int main() { checkEncoder(device.Get(), context.Get(), true); checkEncoder(device.Get(), context.Get(), false); timeFullHd(device.Get(), context.Get()); + checkRepeatedFrames(device.Get(), context.Get()); std::cout << "ran " << g_ran << " tests\n"; if (g_failed != 0) { From dca1bf9d539b442d6d62313dd3327918b460fd40 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Thu, 1 Oct 2026 01:14:27 +0200 Subject: [PATCH 2/2] test(windows): widen the repeat tests' temp paths through the ANSI code page --- electron/native/wgc-capture/src/mf_encoder_color_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp index dad217a1c..1cc20d6b0 100644 --- a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp +++ b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp @@ -298,7 +298,7 @@ void checkRepeatedFrames(ID3D11Device* device, ID3D11DeviceContext* context) { { const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat.mp4"; - const std::wstring widePath(path.begin(), path.end()); + const std::wstring widePath = widen(path); Microsoft::WRL::ComPtr texture; MFEncoder encoder; if (!makeTexture(kWidth, kHeight, texture) || @@ -328,7 +328,7 @@ void checkRepeatedFrames(ID3D11Device* device, ID3D11DeviceContext* context) { { const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat-timing.mp4"; - const std::wstring widePath(path.begin(), path.end()); + const std::wstring widePath = widen(path); Microsoft::WRL::ComPtr texture; MFEncoder encoder; if (!makeTexture(1920, 1080, texture) ||