Auto-inject frames the LLM forgot to embed

Empirically DeepSeek V4 Pro is inconsistent at including
`![Frame N](path)` markdown even when the prompt explicitly asks for
inclusiveness. On EE2022 Stage 3 we observed:

  • 01_04 (Wed): 23 frames extracted → 0 embedded
  • 04_02 (Wed): 3 → 1
  • 10_04 (Fri): 3 → 0
  • 04_03 (Wed): 21 → 3 (~14 % kept)

Strengthening the prompt is unreliable since the LLM's image-decision
is judgment-based. Instead, add a deterministic post-process:
`_ensure_frames_embedded` runs after `_clean_artifacts` and before
translation. For every frame in `img_map` that the draft did not
reference, it appends `![Frame N](path) *(caption)*` at section end,
using the cached vision description (first-sentence-truncated) as the
caption. The downstream `filter_images_pass` still drops junk frames,
so we get a guaranteed-include floor without losing the junk-filter
ceiling.

6 new regression tests cover: missing-frame append, deduplication of
already-referenced frames, no-op on empty img_map, slide-vs-frame
prefix selection, caption fallback to "Frame N" when no description,
and first-sentence caption truncation.

Author: nodeeeeee

electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-note",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "AutoNote — lecture notes generator from Canvas recordings",
   "homepage": "https://github.com/nodeeeeee/Auto-Note",
   "author": {

note_generation.py

@@ -888,6 +888,55 @@ def _clean_artifacts(text: str) -> str:
     return "\n".join(cleaned)
 
 
+def _ensure_frames_embedded(
+    draft: str,
+    slides: list,
+    img_map: dict,
+    img_cache: dict,
+    out_dir: Path,
+    source: str,
+) -> str:
+    """Append any frames the LLM forgot to include in its draft.
+
+    The `_build_chunk_prompt` lists every available frame in the prompt
+    under "Available images", but DeepSeek V4 (and some other models)
+    are inconsistent at actually emitting the `![Frame N](path)` markdown
+    even when explicitly asked. This safety net keeps the contract:
+    every extracted frame is given a chance to surface in the final
+    note. The downstream image-filter pass (`filter_images_pass`) still
+    runs and can drop junk frames — we just guarantee they reach that
+    pass instead of being silently dropped by the LLM.
+    """
+    if not img_map:
+        return draft
+    appended: list[str] = []
+    for s in slides:
+        if s.index not in img_map:
+            continue
+        rel = img_map[s.index].relative_to(out_dir)
+        rel_str = str(rel).replace("\\", "/")
+        # Already cited somewhere in the draft? Skip.
+        if rel_str in draft:
+            continue
+        cache_key = f"page_{s.index}"
+        desc = (img_cache.get(cache_key, "") or "").strip()
+        # First sentence of the description as the caption (max 140 chars).
+        caption = desc[:140]
+        for end in ".。！？!?":
+            idx = caption.find(end)
+            if 25 < idx < len(caption):
+                caption = caption[:idx + 1]
+                break
+        if not caption:
+            caption = f"Frame {s.index + 1}" if source == "screenshare" \
+                else f"Slide {s.index + 1}"
+        prefix = "Frame" if source == "screenshare" else "Slide"
+        appended.append(f"![{prefix} {s.index + 1}]({rel_str}) *({caption})*")
+    if not appended:
+        return draft
+    return draft.rstrip() + "\n\n" + "\n\n".join(appended) + "\n"
+
+
 _BAD_LABEL = re.compile(
     r"^\s*(\d+|[A-Z]{2,4}\d{4}[\s\-].*|CS\d+.*|AY\d+.*|\[.*\]|"
     r".*NUS Confidential.*|.*©\s*CS\d+.*|\(c\)\s*CS\d+.*|Page\s+\d+)\s*$",
@@ -1152,6 +1201,17 @@ def generate_section(
     # Strip pipeline artifacts that may have leaked into the draft
     draft = _clean_artifacts(draft)
 
+    # Frame-completeness safety net: DeepSeek V4 (and other models under
+    # certain phrasings) is inconsistent at embedding `![Frame N](...)`
+    # markdown even when the prompt asks for it — empirically the LLM
+    # keeps anywhere from 0–100% of the available frames per chunk.
+    # Auto-append any frame in img_map that didn't make it into the draft
+    # so we never silently lose the visual content the user asked us to
+    # extract. The downstream image-filter pass still drops junk frames.
+    draft = _ensure_frames_embedded(
+        draft, chunk, img_map, ld.img_cache, out_dir, ld.source,
+    )
+
     # Translate to target language if not English
     if NOTE_LANGUAGE != "en" and draft:
         lang = _LANG_NAMES.get(NOTE_LANGUAGE, NOTE_LANGUAGE)

test/test_frame_injection.py

@@ -0,0 +1,139 @@
+"""
+Regression tests for the v1.0.7 frame-injection safety net.
+
+DeepSeek V4 Pro (and some other models) is inconsistent at embedding
+`![Frame N](path)` markdown even when the prompt explicitly asks for
+it — empirically observed dropping 23-of-23 frames on one EE2022
+lecture and 0/3 on another. `_ensure_frames_embedded` post-processes
+the draft and appends any frame in `img_map` that the LLM didn't cite.
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+PROJECT_DIR = Path(__file__).parent.parent
+sys.path.insert(0, str(PROJECT_DIR))
+
+
+def _make_slide(idx, label="Slide"):
+    return SimpleNamespace(index=idx, label=label, text="", word_count=0,
+                           has_code=False)
+
+
+class TestEnsureFramesEmbedded:
+    def test_appends_missing_frames_with_caption(self, tmp_path):
+        from note_generation import _ensure_frames_embedded
+        out_dir = tmp_path
+        # Create dummy frame paths under out_dir
+        frames_dir = out_dir / "images" / "L01_Foo"
+        frames_dir.mkdir(parents=True)
+        f1 = frames_dir / "frame_001.png"; f1.touch()
+        f2 = frames_dir / "frame_002.png"; f2.touch()
+
+        slides = [_make_slide(0), _make_slide(1)]
+        img_map = {0: f1, 1: f2}
+        img_cache = {
+            "page_0": "Renewable Energy Integration – Power Electronic Convertors. Diagram.",
+            "page_1": "Synchronous reactance equivalent circuit.",
+        }
+        # Draft with NO frame markdown — the bug case.
+        draft = "Some Chinese 内容 about the topic."
+
+        out = _ensure_frames_embedded(
+            draft, slides, img_map, img_cache, out_dir, source="screenshare",
+        )
+        assert "![Frame 1](" in out
+        assert "![Frame 2](" in out
+        assert "Renewable Energy Integration" in out  # caption preserved
+        assert "Synchronous reactance" in out
+
+    def test_keeps_already_referenced_frames(self, tmp_path):
+        from note_generation import _ensure_frames_embedded
+        out_dir = tmp_path
+        frames_dir = out_dir / "images" / "L01_Foo"
+        frames_dir.mkdir(parents=True)
+        f1 = frames_dir / "frame_001.png"; f1.touch()
+        f2 = frames_dir / "frame_002.png"; f2.touch()
+
+        slides = [_make_slide(0), _make_slide(1)]
+        img_map = {0: f1, 1: f2}
+        img_cache = {"page_0": "first", "page_1": "second"}
+        # Draft already references frame 1 — don't duplicate it.
+        draft = "Foo. ![Frame 1](images/L01_Foo/frame_001.png) *(first)* Bar."
+
+        out = _ensure_frames_embedded(
+            draft, slides, img_map, img_cache, out_dir, source="screenshare",
+        )
+        # Frame 1 not duplicated
+        assert out.count("![Frame 1](") == 1
+        # Frame 2 appended
+        assert "![Frame 2](" in out
+
+    def test_no_op_when_img_map_empty(self, tmp_path):
+        from note_generation import _ensure_frames_embedded
+        slides = [_make_slide(0), _make_slide(1)]
+        draft = "Some prose."
+        out = _ensure_frames_embedded(
+            draft, slides, {}, {}, tmp_path, source="screenshare",
+        )
+        assert out == draft
+
+    def test_uses_slide_prefix_for_slide_pdf_source(self, tmp_path):
+        from note_generation import _ensure_frames_embedded
+        out_dir = tmp_path
+        slides_dir = out_dir / "images" / "L01_Foo"
+        slides_dir.mkdir(parents=True)
+        s1 = slides_dir / "slide_001.png"; s1.touch()
+
+        slides = [_make_slide(0)]
+        img_map = {0: s1}
+        img_cache = {"page_0": "Architecture diagram."}
+        draft = "Prose without images."
+
+        out = _ensure_frames_embedded(
+            draft, slides, img_map, img_cache, out_dir, source="slides",
+        )
+        # Slide-PDF source uses "Slide N" prefix, not "Frame N"
+        assert "![Slide 1](" in out
+        assert "![Frame 1](" not in out
+
+    def test_caption_falls_back_to_index_when_no_description(self, tmp_path):
+        from note_generation import _ensure_frames_embedded
+        out_dir = tmp_path
+        frames_dir = out_dir / "images" / "L01_Foo"
+        frames_dir.mkdir(parents=True)
+        f1 = frames_dir / "frame_007.png"; f1.touch()
+
+        slides = [_make_slide(6)]   # 0-based index 6 → frame_007
+        img_map = {6: f1}
+        # Empty cache — no description available.
+        out = _ensure_frames_embedded(
+            "draft", slides, img_map, {}, out_dir, source="screenshare",
+        )
+        assert "![Frame 7](" in out
+        assert "*(Frame 7)*" in out
+
+    def test_long_description_truncated_at_first_sentence(self, tmp_path):
+        from note_generation import _ensure_frames_embedded
+        out_dir = tmp_path
+        frames_dir = out_dir / "images" / "L01_Foo"
+        frames_dir.mkdir(parents=True)
+        f1 = frames_dir / "frame_001.png"; f1.touch()
+
+        slides = [_make_slide(0)]
+        img_map = {0: f1}
+        # Two sentences — only the first should appear in the caption.
+        img_cache = {"page_0": "First sentence about reactance. Second sentence about flux."}
+
+        out = _ensure_frames_embedded(
+            "draft", slides, img_map, img_cache, out_dir, source="screenshare",
+        )
+        # Caption text appears as `*(...)*` — find it
+        import re
+        m = re.search(r"\*\(([^)]+)\)\*", out)
+        assert m is not None
+        caption = m.group(1)
+        assert "First sentence" in caption
+        assert "Second sentence" not in caption