perf(windows): repeat the last frame instead of reading back an unchanged screen

The video writer read the full GPU frame back on every tick, even when
WGC delivered nothing new: a CopyResource, a Map that waits on the GPU,
and the BGRA to NV12 conversion, for a picture it already had. On a
static screen, most of a demo, that was every tick.

When no new WGC frame arrived and no new camera frame is drawn into the
picture, the encoder now hands over a new sample on the last buffer.
Measured at 1080p: 5.0 ms per readback, under 0.001 ms per repeat. The
DXGI path and the legacy callback path read back as before.

Fixes #925

Author: EtienneLescot

electron/native/wgc-capture/src/main.cpp

@@ -1098,6 +1098,10 @@ int wmain(int argc, wchar_t* argv[]) {
             Microsoft::WRL::ComPtr<IMFSample> webcamSample;
             bool hasVideoSample = false;
             bool hasWebcamSample = false;
+            // Whether the picture this tick encodes differs from the last one:
+            // a new WGC frame, or a new camera frame drawn into it. The legacy
+            // callback path cannot tell, so it always reads back.
+            bool pictureChanged = legacyFrameCallback;
 
             std::unique_lock<std::timed_mutex> legacyLock;
             {
@@ -1136,6 +1140,7 @@ int wmain(int argc, wchar_t* argv[]) {
                     ID3D11Texture2D* wgcTexture = nullptr;
                     int64_t wgcTimestampHns = 0;
                     const bool gotFrame = session.tryGetNextFrame(&wgcTexture, &wgcTimestampHns);
+                    pictureChanged = gotFrame;
                     if (gotFrame) {
                         if (!latestFrameTexture) {
                             D3D11_TEXTURE2D_DESC desc{};
@@ -1179,6 +1184,7 @@ int wmain(int argc, wchar_t* argv[]) {
                         latestWebcamHeight = candidateWebcamFrame.height;
                         latestWebcamSequence = candidateWebcamFrame.sequence;
                         hasVisibleWebcamFrame = true;
+                        pictureChanged = pictureChanged || !writeSeparateWebcam;
                     }
                 }
                 const BgraFrameView webcamFrame{
@@ -1272,7 +1278,12 @@ int wmain(int argc, wchar_t* argv[]) {
                     // this struct's request: it falls back to the CPU path on
                     // its own when the GPU path does not fit the machine.
                     bool captured = false;
-                    if (usesDxgiInput) {
+                    if (!usesDxgiInput && !pictureChanged &&
+                        encoder.repeatLastVideoSample(frameTimestampHns, videoSample)) {
+                        // Nothing new to read back (#925): the static screen,
+                        // most of a demo, costs no GPU copy and no conversion.
+                        captured = true;
+                    } else if (usesDxgiInput) {
                         captured = encoder.captureDxgiSample(
                             latestFrameTexture.Get(),
                             frameTimestampHns,

electron/native/wgc-capture/src/mf_encoder.cpp

@@ -1761,6 +1761,26 @@ bool MFEncoder::captureVideoSample(
     sample->SetSampleTime(sampleTime);
     sample->SetSampleDuration(sampleDuration);
 
+    lastVideoBuffer_ = buffer;
+    outSample = sample;
+    return true;
+}
+
+bool MFEncoder::repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr<IMFSample>& outSample) {
+    outSample.Reset();
+    if (!lastVideoBuffer_) {
+        return false;
+    }
+    const int64_t sampleDuration = 10'000'000LL / fps_;
+    Microsoft::WRL::ComPtr<IMFSample> sample;
+    if (!succeeded(MFCreateSample(&sample), "MFCreateSample(repeat)")) {
+        return false;
+    }
+    // Shared, not copied: a buffer is never written again once its sample is
+    // built, so two samples can hand the encoder the same bytes.
+    sample->AddBuffer(lastVideoBuffer_.Get());
+    sample->SetSampleTime(nextSampleTime(timestampHns, sampleDuration));
+    sample->SetSampleDuration(sampleDuration);
     outSample = sample;
     return true;
 }

electron/native/wgc-capture/src/mf_encoder.h

@@ -138,6 +138,11 @@ class MFEncoder {
         int64_t timestampHns,
         const BgraFrameView* webcamFrame,
         Microsoft::WRL::ComPtr<IMFSample>& outSample);
+    // The last frame captureVideoSample read back, again, at a new time: for a
+    // tick on which WGC delivered nothing new. No GPU copy, no Map, no
+    // conversion -- the readback a static screen used to pay on every tick
+    // (getopenscreen/openscreen#925). False before any frame was captured.
+    bool repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr<IMFSample>& outSample);
     bool captureDxgiSample(
         ID3D11Texture2D* texture,
         int64_t timestampHns,
@@ -257,6 +262,7 @@ class MFEncoder {
     // The BGRA frame when it has to be drawn on or rescaled before the NV12
     // conversion; reused so a frame does not allocate one.
     std::vector<BYTE> bgraScratch_;
+    Microsoft::WRL::ComPtr<IMFMediaBuffer> lastVideoBuffer_;
     int width_ = 0;
     int height_ = 0;
     int fps_ = 60;

electron/native/wgc-capture/src/mf_encoder_color_test.cpp

@@ -258,6 +258,97 @@ void timeFullHd(ID3D11Device* device, ID3D11DeviceContext* context) {
               << captureMs / frames << " ms, submit " << submitMs / frames << " ms per frame" << std::endl;
 }
 
+// A tick with no new WGC frame repeats the last buffer instead of reading the
+// texture back again (getopenscreen/openscreen#925). The file must still hold
+// one frame per tick, each the captured picture; and the repeat must cost next
+// to nothing next to a readback, or it is not worth having.
+void checkRepeatedFrames(ID3D11Device* device, ID3D11DeviceContext* context) {
+    const auto makeTexture = [&](int width, int height, Microsoft::WRL::ComPtr<ID3D11Texture2D>& texture) {
+        std::vector<BYTE> bgra(static_cast<size_t>(width) * height * 4);
+        for (int y = 0; y < height; y += 1) {
+            for (int x = 0; x < width; x += 1) {
+                BYTE* pixel = &bgra[(static_cast<size_t>(y) * width + x) * 4];
+                pixel[2] = 255;  // solid red
+                pixel[3] = 255;
+            }
+        }
+        D3D11_TEXTURE2D_DESC desc{};
+        desc.Width = width;
+        desc.Height = height;
+        desc.MipLevels = 1;
+        desc.ArraySize = 1;
+        desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
+        desc.SampleDesc.Count = 1;
+        desc.Usage = D3D11_USAGE_DEFAULT;
+        desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
+        D3D11_SUBRESOURCE_DATA initial{bgra.data(), static_cast<UINT>(width * 4), 0};
+        return SUCCEEDED(device->CreateTexture2D(&desc, &initial, &texture));
+    };
+    char tempDir[MAX_PATH]{};
+    GetTempPathA(MAX_PATH, tempDir);
+
+    {
+        const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat.mp4";
+        const std::wstring widePath(path.begin(), path.end());
+        Microsoft::WRL::ComPtr<ID3D11Texture2D> texture;
+        MFEncoder encoder;
+        if (!makeTexture(kWidth, kHeight, texture) ||
+            !encoder.initialize(widePath, kWidth, kHeight, 30, 2'000'000, device, context, nullptr, {})) {
+            expect("repeat-setup", false, "texture or encoder");
+            return;
+        }
+        Microsoft::WRL::ComPtr<IMFSample> sample;
+        bool wrote = !encoder.repeatLastVideoSample(0, sample) &&  // nothing to repeat yet
+            encoder.captureVideoSample(texture.Get(), 0, nullptr, sample) && encoder.submitVideoSample(sample.Get());
+        for (int i = 1; i < kFrames && wrote; i += 1) {
+            wrote = encoder.repeatLastVideoSample(static_cast<int64_t>(i) * 333'333, sample) &&
+                    encoder.submitVideoSample(sample.Get());
+        }
+        expect("repeat-writes", wrote && encoder.finalize(), "write or finalize failed");
+        const std::string frames = run(
+            "ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames "
+            "-of default=nw=1:nk=1 \"" + path + "\"");
+        const std::string yuv = run(
+            "ffmpeg -v error -sseof -0.2 -i \"" + path + "\" -frames:v 1 -f rawvideo -pix_fmt yuv420p -");
+        const int luma = yuv.empty() ? -1 : static_cast<unsigned char>(yuv[static_cast<size_t>(kHeight / 2) * kWidth + kWidth / 2]);
+        std::cout << "REPEAT_RAW frames=" << std::atoi(frames.c_str()) << " last-frame Y=" << luma << std::endl;
+        expect("repeat-keeps-one-frame-per-tick", std::atoi(frames.c_str()) == kFrames, frames);
+        expect("repeat-keeps-the-picture", std::abs(luma - 63) <= 3, "Y=" + std::to_string(luma));
+        DeleteFileA(path.c_str());
+    }
+
+    {
+        const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat-timing.mp4";
+        const std::wstring widePath(path.begin(), path.end());
+        Microsoft::WRL::ComPtr<ID3D11Texture2D> texture;
+        MFEncoder encoder;
+        if (!makeTexture(1920, 1080, texture) ||
+            !encoder.initialize(widePath, 1920, 1080, 60, 18'000'000, device, context, nullptr, {})) {
+            return;
+        }
+        double readbackMs = 0.0;
+        double repeatMs = 0.0;
+        constexpr int rounds = 60;
+        for (int i = 0; i < rounds; i += 1) {
+            Microsoft::WRL::ComPtr<IMFSample> sample;
+            const auto start = std::chrono::steady_clock::now();
+            encoder.captureVideoSample(texture.Get(), static_cast<int64_t>(2 * i) * 166'667, nullptr, sample);
+            const auto read = std::chrono::steady_clock::now();
+            encoder.submitVideoSample(sample.Get());
+            const auto repeatStart = std::chrono::steady_clock::now();
+            encoder.repeatLastVideoSample(static_cast<int64_t>(2 * i + 1) * 166'667, sample);
+            const auto repeated = std::chrono::steady_clock::now();
+            encoder.submitVideoSample(sample.Get());
+            readbackMs += std::chrono::duration<double, std::milli>(read - start).count();
+            repeatMs += std::chrono::duration<double, std::milli>(repeated - repeatStart).count();
+        }
+        encoder.finalize();
+        DeleteFileA(path.c_str());
+        std::cout << "REPEAT_RAW 1080p readback " << readbackMs / rounds << " ms, repeat " << repeatMs / rounds
+                  << " ms per frame" << std::endl;
+    }
+}
+
 } // namespace
 
 int main() {
@@ -285,6 +376,7 @@ int main() {
     checkEncoder(device.Get(), context.Get(), true);
     checkEncoder(device.Get(), context.Get(), false);
     timeFullHd(device.Get(), context.Get());
+    checkRepeatedFrames(device.Get(), context.Get());
 
     std::cout << "ran " << g_ran << " tests\n";
     if (g_failed != 0) {