Skip to content

Commit a6d4dfa

Browse files
nodeeeeeeclaude
andcommitted
Fix matching quality and suppress position_ids warning
Matching fix: - BGE-M3 pre-match now uses only the specific file it matched, not all files with the same lecture number. Previously, matching to "Lecture1 Intro With notes.pdf" expanded to 5 files (original, annotated, review copies), producing poor alignment across 90+ combined duplicate slides. position_ids fix: - Suppress the safetensors LOAD REPORT ("embeddings.position_ids UNEXPECTED") by redirecting OS-level file descriptors during model loading. The warning comes from C/Rust code that bypasses Python's sys.stdout, so warnings.catch_warnings() and logging don't help. Language fix (from previous commit, confirmed working): - --language zh auto-enables force-regenerate - Tested: notes generated in Chinese with correct prompts Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent bc42177 commit a6d4dfa

1 file changed

Lines changed: 37 additions & 8 deletions

File tree

semantic_alignment.py

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -404,12 +404,28 @@ def load_slides(path: Path,
404404

405405
def get_embedder():
406406
"""Load sentence-transformer model once; use GPU if available."""
407+
import warnings
407408
from sentence_transformers import SentenceTransformer
408409
import torch
409410
device = "cuda" if torch.cuda.is_available() else "cpu"
410411
print(f" [embed] Loading {EMBED_MODEL} on {device} ...", flush=True)
411412
try:
412-
model = SentenceTransformer(EMBED_MODEL, device=device)
413+
# Suppress the safetensors LOAD REPORT (position_ids UNEXPECTED).
414+
# It writes directly to fd 1/2, so we must redirect at the OS level.
415+
import os as _os
416+
_devnull = _os.open(_os.devnull, _os.O_WRONLY)
417+
_saved_stdout = _os.dup(1)
418+
_saved_stderr = _os.dup(2)
419+
_os.dup2(_devnull, 1)
420+
_os.dup2(_devnull, 2)
421+
try:
422+
model = SentenceTransformer(EMBED_MODEL, device=device)
423+
finally:
424+
_os.dup2(_saved_stdout, 1)
425+
_os.dup2(_saved_stderr, 2)
426+
_os.close(_devnull)
427+
_os.close(_saved_stdout)
428+
_os.close(_saved_stderr)
413429
except Exception as e:
414430
print(f" [embed] Failed to load model: {e}", flush=True)
415431
raise
@@ -1486,8 +1502,21 @@ def suggest_matches(
14861502
import torch
14871503
device = "cuda" if torch.cuda.is_available() else "cpu"
14881504
print(f" [match] Loading {model_name} on {device}...")
1489-
embedder = SentenceTransformer(model_name, device=device,
1490-
trust_remote_code=True)
1505+
import os as _os
1506+
_devnull = _os.open(_os.devnull, _os.O_WRONLY)
1507+
_saved_out = _os.dup(1)
1508+
_saved_err = _os.dup(2)
1509+
_os.dup2(_devnull, 1)
1510+
_os.dup2(_devnull, 2)
1511+
try:
1512+
embedder = SentenceTransformer(model_name, device=device,
1513+
trust_remote_code=True)
1514+
finally:
1515+
_os.dup2(_saved_out, 1)
1516+
_os.dup2(_saved_err, 2)
1517+
_os.close(_devnull)
1518+
_os.close(_saved_out)
1519+
_os.close(_saved_err)
14911520
all_texts = cap_texts + slide_texts
14921521
embs = embedder.encode(
14931522
all_texts,
@@ -1634,11 +1663,11 @@ def process_course(course_id: int | str, use_jina: bool = False,
16341663
for stem, rel in raw.items():
16351664
sp = course_dir / rel
16361665
if sp.exists():
1637-
num = _lec_num(sp)
1638-
if num is not None and num in slides_by_num:
1639-
bge_matches[stem] = sorted(slides_by_num[num])
1640-
else:
1641-
bge_matches[stem] = [sp]
1666+
# Use only the specific file BGE-M3 picked — don't expand to
1667+
# all files with the same lecture number, as that would include
1668+
# duplicate versions (annotated, review, with-notes copies) and
1669+
# degrade alignment quality.
1670+
bge_matches[stem] = [sp]
16421671
if bge_matches:
16431672
print(f" [bge-m3] Pre-matched {len(bge_matches)} video(s) to slides")
16441673
except Exception as e:

0 commit comments

Comments
 (0)