From 9bb4a17837f0b051d0e7ac2fb1153ca1c86dcda7 Mon Sep 17 00:00:00 2001 From: dull bird <1155115927@link.cuhk.edu.hk> Date: Sat, 25 Jul 2026 00:42:06 +0800 Subject: [PATCH 1/4] feat: experimental keyless CLI backend (Claude Code) and subtitle+frame video analysis - Add core/llm_cli.py: drive Claude Code as an LLM backend via session-resume multi-turn and a text tool-call protocol. Activated by R2S_LLM_BACKEND=cli. - Wire dispatch in core/llm.py so call_azure_openai and call_llm route to the CLI backend without changing any call sites. - Add core/analyzer_cli_video.py: R2S_VIDEO_BACKEND=cli analyzes YouTube videos using yt-dlp subtitles + ffmpeg keyframes instead of Gemini. - Make google-genai import lazy in core/analyzer.py so the CLI video path can run without the Gemini SDK installed. - Add skill/Resource2Skill/SKILL.md connector for agents to drive cli.py. - Update README.md and .env.example with no-API-key setup instructions. --- .env.example | 14 ++ README.md | 24 +++ core/analyzer.py | 31 ++- core/analyzer_cli_video.py | 338 +++++++++++++++++++++++++++++++ core/llm.py | 29 +++ core/llm_cli.py | 371 ++++++++++++++++++++++++++++++++++ skill/Resource2Skill/SKILL.md | 124 ++++++++++++ 7 files changed, 928 insertions(+), 3 deletions(-) create mode 100644 core/analyzer_cli_video.py create mode 100644 core/llm_cli.py create mode 100644 skill/Resource2Skill/SKILL.md diff --git a/.env.example b/.env.example index 727d35da..76c55980 100644 --- a/.env.example +++ b/.env.example @@ -21,3 +21,17 @@ # OPENAI_API_KEY= # GEMINI_API_KEY= # GOOGLE_API_KEY= + +# --------------------------------------------------------------------------- +# Local agent CLI backend (no API keys) +# --------------------------------------------------------------------------- +# Set R2S_LLM_BACKEND=cli to route LLM calls through a locally installed +# agent CLI instead of Azure OpenAI. Requires that CLI to be authenticated +# via its own login flow (e.g. `claude login`). +# R2S_LLM_BACKEND=cli +# R2S_CLI_TOOL=claude # claude | codex | kimi | omp +# R2S_CLI_MODEL= # optional model override passed to the CLI + +# Set R2S_VIDEO_BACKEND=cli to analyze YouTube videos using local yt-dlp +# subtitles + ffmpeg keyframes instead of Gemini native video understanding. +# R2S_VIDEO_BACKEND=cli diff --git a/README.md b/README.md index 00577fb4..18aa53f5 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,30 @@ AZURE_OPENAI_API_KEY= Per-model overrides (e.g. `AZURE_OPENAI_ENDPOINT_54`, `AZURE_OPENAI_DEPLOYMENT_54`) are in `.env.example`. +## Running without API keys + +If you have a local agent CLI installed and authenticated, you can run most of +Resource2Skill without Azure OpenAI or Gemini keys. + +```bash +cp .env.example .env +# edit .env: set R2S_LLM_BACKEND=cli and R2S_CLI_TOOL=claude + +# or export for one shell +export R2S_LLM_BACKEND=cli +export R2S_CLI_TOOL=claude + +# video analysis also works without Gemini by using yt-dlp subtitles + ffmpeg keyframes +export R2S_VIDEO_BACKEND=cli +``` + +Tested: `claude` supports multi-turn session resume, a text-based tool-call +protocol, and reading local image files in headless mode. `codex`, `kimi`, and +`omp` backends are not yet implemented. + +If you are an agent reading this repo, use `skill/Resource2Skill/SKILL.md` as +the connector for driving `cli.py`. + ## Quick Start ```bash diff --git a/core/analyzer.py b/core/analyzer.py index 1253e93f..58fa706d 100644 --- a/core/analyzer.py +++ b/core/analyzer.py @@ -14,8 +14,14 @@ import time import logging from pathlib import Path -from google import genai -from google.genai import types +try: + from google import genai + from google.genai import types +except ImportError as _genai_import_err: # pragma: no cover - optional dependency + genai = None # type: ignore[assignment] + types = None # type: ignore[assignment] + +from core.analyzer_cli_video import analyze_video_cli log = logging.getLogger("analyzer") @@ -41,7 +47,12 @@ def _load_prompt(path: Path | str | None = None) -> str: return p.read_text(encoding="utf-8") -def _make_client(api_key: str | None = None) -> genai.Client: +def _make_client(api_key: str | None = None): + if genai is None: + raise RuntimeError( + "google-genai package is not installed. " + "Install it or use R2S_VIDEO_BACKEND=cli for keyless video analysis." + ) key = api_key or os.environ.get("GEMINI_API_KEY") if not key: raise ValueError( @@ -63,6 +74,10 @@ def analyze_video( ) -> str: """Analyze a YouTube video and return a skill extraction in Markdown. + Defaults to Gemini native video understanding. When ``R2S_VIDEO_BACKEND=cli`` + is set, falls back to a local, keyless pipeline: yt-dlp subtitles + + ffmpeg keyframes + local agent CLI (Claude Code) reasoning. + Args: video_url: Public YouTube URL. model: Gemini model to use. @@ -76,6 +91,16 @@ def analyze_video( Returns: Raw Markdown analysis from the model. """ + if os.environ.get("R2S_VIDEO_BACKEND", "").strip().lower() == "cli": + return analyze_video_cli( + video_url, + prompt_path=prompt_path, + prompt_text=prompt_text, + extra_instructions=extra_instructions, + start_offset=start_offset, + end_offset=end_offset, + ) + client = _make_client(api_key) if prompt_text: diff --git a/core/analyzer_cli_video.py b/core/analyzer_cli_video.py new file mode 100644 index 00000000..31b2ffa5 --- /dev/null +++ b/core/analyzer_cli_video.py @@ -0,0 +1,338 @@ +""" +core/analyzer_cli_video.py + +Keyless video analysis backend for Resource2Skill. + +Used when ``R2S_VIDEO_BACKEND=cli`` is set. It replaces the Gemini native +YouTube-URL understanding with a local pipeline: + + 1. yt-dlp downloads the video's auto-generated subtitles (VTT). + 2. ffmpeg extracts a small set of key frames from the video. + 3. A local agent CLI (currently Claude Code via core.llm_cli) reads the + subtitle transcript + frame image paths and produces the same Markdown + skill analysis that ``analyze_video`` would return. + +The transcript parser handles YouTube's rolling-caption format by deduplicating +overlapping cue text, preserving timestamps so the model can map statements to +video moments. +""" +from __future__ import annotations + +import glob +import json +import logging +import os +import re +import subprocess +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from core.analyzer import analyze_video + +log = logging.getLogger("analyzer") + +# --------------------------------------------------------------------------- +# Subtitle acquisition and parsing +# --------------------------------------------------------------------------- + + +def _run_ytdlp(*args, timeout: int = 120) -> subprocess.CompletedProcess: + """Run a yt-dlp command and return the result.""" + cmd = ["yt-dlp", *args] + log.debug("yt-dlp command: %s", " ".join(cmd)) + return subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + ) + + +def _get_duration_sec(video_url: str) -> int: + """Get video duration in seconds without downloading the video.""" + try: + result = _run_ytdlp("--print", "duration", "--skip-download", "--quiet", video_url) + if result.returncode == 0 and result.stdout.strip(): + return int(float(result.stdout.strip().splitlines()[0])) + except Exception as e: + log.warning("Could not determine video duration: %s", e) + return 0 + + +def _fetch_vtt(video_url: str, output_dir: Path, *, start_offset: str | None, end_offset: str | None) -> Path | None: + """Download auto-generated subtitles; return the VTT file path or None.""" + # Prefer English, then any Chinese variant, then auto-detect by leaving a + # fallback that accepts whatever yt-dlp can find. + lang_list = ["en.*", "zh.*", "zh-Hans.*"] + base = output_dir / "video" + try: + result = _run_ytdlp( + "--write-auto-subs", + "--write-subs", + "--sub-langs", ",".join(lang_list), + "--skip-download", + "--sub-format", "vtt", + "--convert-subs", "vtt", + "--output", str(base) + ".%(ext)s", + video_url, + timeout=120, + ) + except subprocess.TimeoutExpired: + log.warning("yt-dlp subtitle fetch timed out") + return None + + # yt-dlp names files like video.en.auto.vtt or video.zh-Hans.vtt + candidates = sorted(glob.glob(str(base) + "*.vtt")) + if not candidates: + log.warning("No VTT subtitles found for %s", video_url) + return None + return Path(candidates[0]) + + +def _parse_vtt(vtt_path: Path) -> str: + """Parse a VTT file into a timestamped transcript with rolling dedup. + + YouTube auto-generated captions emit overlapping cues where cue N is often a + suffix of cue N-1 plus a few new words. This function strips cue tags and + merges the rolling window so each word appears only once. + """ + text = vtt_path.read_text(encoding="utf-8", errors="ignore") + + # Strip WEBVTT header and any NOTE blocks / STYLE blocks. + body = re.sub(r"WEBVTT[^\n]*\n", "", text, count=1) + body = re.sub(r"(?m)^NOTE[\s\S]*?(?=\n\n|\n[A-Z0-9])", "", body) + body = re.sub(r"(?m)^STYLE[\s\S]*?(?=\n\n|\n[A-Z0-9])", "", body) + + timing_re = re.compile(r"^(\d{1,2}:)?\d{1,2}:\d{2}(\.\d{3})?\s+-->") + + cues: list[tuple[int, str]] = [] + blocks = re.split(r"\n\s*\n", body) + for block in blocks: + lines = [ln.strip() for ln in block.splitlines() if ln.strip()] + if not lines: + continue + # The timing line is the first line that looks like a cue arrow. + timing_line: str | None = None + for ln in lines: + if timing_re.match(ln): + timing_line = ln + break + if timing_line is None: + continue + # Cue body is everything after the timing line. + idx = lines.index(timing_line) + cue_body = "\n".join(lines[idx + 1:]) + # Strip cue tags like , , , positioning tags, and empty brackets. + cue_body = re.sub(r"<[^>]+>", "", cue_body) + cue_body = re.sub(r"\[\s*\]", "", cue_body) + cue_body = " ".join(cue_body.split()) + if not cue_body: + continue + # Parse the start timestamp (first token on timing line). + start_token = timing_line.split()[0] + parts = start_token.split(":") + try: + if len(parts) == 2: + mins = int(parts[0]) + secs = float(parts[1]) + start_sec = mins * 60 + int(secs) + elif len(parts) == 3: + hrs = int(parts[0]) + mins = int(parts[1]) + secs = float(parts[2]) + start_sec = hrs * 3600 + mins * 60 + int(secs) + else: + start_sec = 0 + except ValueError: + continue + cues.append((start_sec, cue_body)) + + if not cues: + return "" + + # Rolling dedup: build one continuous transcript by removing text that has + # already appeared at the end of the previous cue. + merged: list[str] = [] + prev = "" + for sec, raw in cues: + # Normalize for overlap detection + a = prev.strip() + b = raw.strip() + # Case 1: new cue is a suffix of previous cue (YouTube rarely does this) + if b and a.endswith(b): + continue + # Case 2: new cue starts with previous cue (common in rolling captions) + if b.startswith(a): + suffix = b[len(a):].strip() + if suffix: + merged.append(f"[{_fmt_time(sec)}] {suffix}") + prev = b + continue + # Case 3: overlap smaller than full previous cue: find longest common + # suffix/prefix match. + overlap_len = 0 + for i in range(min(len(a), len(b)), 0, -1): + if a.endswith(b[:i]): + overlap_len = i + break + if overlap_len: + suffix = b[overlap_len:].strip() + if suffix: + merged.append(f"[{_fmt_time(sec)}] {suffix}") + prev = b + else: + merged.append(f"[{_fmt_time(sec)}] {b}") + prev = b + + return "\n".join(merged) + + +def _fmt_time(seconds: int) -> str: + m, s = divmod(seconds, 60) + h, m = divmod(m, 60) + if h: + return f"{h}:{m:02d}:{s:02d}" + return f"{m}:{s:02d}" + + +# --------------------------------------------------------------------------- +# Frame extraction helper +# --------------------------------------------------------------------------- + + +def _uniform_timestamps(duration: int, n: int = 8) -> list[dict]: + """Return n evenly spaced timestamps across the video.""" + if duration <= 0 or n <= 0: + return [] + if n == 1: + return [{"seconds": max(1, duration // 2), "description": "midpoint"}] + return [ + {"seconds": int(duration * i / (n + 1)), "description": f"stage_{int(100*i/(n+1))}pct"} + for i in range(1, n + 1) + ] + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def analyze_video_cli( + video_url: str, + *, + prompt_path: str | Path | None = None, + prompt_text: str | None = None, + extra_instructions: str = "", + start_offset: str | None = None, + end_offset: str | None = None, + max_frames: int = 6, +) -> str: + """Analyze a video using only local tools and a local agent CLI.""" + from core.llm_cli import call_cli + from core.analyzer import _load_prompt + + if prompt_text: + text = prompt_text + else: + text = _load_prompt(prompt_path) + if extra_instructions: + text += "\n\n" + extra_instructions + + # Determine the effective time range so we can place frames correctly. + start_sec = _parse_offset(start_offset) if start_offset else 0 + full_duration = _get_duration_sec(video_url) + end_sec = _parse_offset(end_offset) if end_offset else full_duration + effective_duration = max(0, end_sec - start_sec) + log.info( + "CLI video backend: duration=%ss, window=[%ss, %ss]", + full_duration, start_sec, end_sec, + ) + + with tempfile.TemporaryDirectory(prefix="r2s_video_cli_") as tmpdir: + tmp = Path(tmpdir) + + # 1. Subtitles + transcript = "" + vtt_path = _fetch_vtt(video_url, tmp, start_offset=start_offset, end_offset=end_offset) + if vtt_path: + transcript = _parse_vtt(vtt_path) + log.info("Parsed transcript length: %d chars", len(transcript)) + else: + log.warning("No subtitle transcript available; analysis will rely on frames only") + + # 2. Key frames (extract to the same temp dir) + timestamps = _uniform_timestamps(effective_duration, n=max_frames) + if start_sec: + timestamps = [ + {"seconds": start_sec + ts["seconds"], "description": ts["description"]} + for ts in timestamps + ] + # Imported here to avoid circular import at module load. + from core.analyzer import extract_frames + + frames = extract_frames(video_url, timestamps, tmp, max_frames=max_frames) if timestamps else [] + log.info("Extracted %d frames", len(frames)) + + # 3. Build prompt + transcript_block = ( + "## Video Subtitle Transcript (timestamped)\n\n" + transcript + if transcript + else "## Video Subtitle Transcript\n\n[No subtitles available for this video.]" + ) + frame_block = "## Key Frames Extracted\n\n" + if frames: + for f in frames: + abs_path = str(tmp / f["path"]) + frame_block += f"- `{abs_path}` at t={_fmt_time(f['seconds'])} ({f['description']})\n" + frame_block += ( + "\nRead the image files above to understand the visual state at each timestamp.\n" + ) + else: + frame_block += "[No frames extracted.]\n" + + user_prompt = ( + f"{text}\n\n{transcript_block}\n\n{frame_block}\n\n" + "Produce the Markdown skill analysis following the instructions in the distiller prompt.\n" + "Include the 'Key Frame Timestamps' table using the exact timestamps shown above." + ) + + system_prompt = ( + "You are a tutorial-to-skill distiller. You have access to a Read tool that can open local image files. " + "Use it when the visual state matters. Return ONLY the Markdown analysis." + ) + + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ] + + try: + msg = call_cli(messages, timeout=600) + return msg.get("content", "") + except Exception as e: + log.error("CLI video analysis failed: %s", e) + raise + + +def _parse_offset(offset: str) -> int: + """Parse '120s' or '2m' or '1:30' into seconds.""" + offset = offset.strip().lower() + # try explicit suffix + m = re.match(r"^(\d+(?:\.\d+)?)\s*(s|sec|m|min|h|hr)?$", offset) + if m: + value = float(m.group(1)) + unit = m.group(2) or "s" + if unit in ("h", "hr"): + return int(value * 3600) + if unit in ("m", "min"): + return int(value * 60) + return int(value) + # try h:mm:ss or m:ss + parts = offset.split(":") + if len(parts) == 2: + return int(parts[0]) * 60 + int(parts[1]) + if len(parts) == 3: + return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2]) + raise ValueError(f"Cannot parse offset: {offset}") diff --git a/core/llm.py b/core/llm.py index 9fa32538..8a175996 100644 --- a/core/llm.py +++ b/core/llm.py @@ -188,6 +188,11 @@ class LLMError(Exception): """Raised when all LLM call attempts fail.""" +def _use_cli_backend() -> bool: + """Route LLM calls through a local agent CLI (no API keys) when set.""" + return os.environ.get("R2S_LLM_BACKEND", "").strip().lower() == "cli" + + def _chat_to_responses_input(messages: list[dict]) -> list[dict]: """Translate chat-format messages into Responses API input items. @@ -439,6 +444,18 @@ def call_azure_openai( Raises: LLMError: If all retries are exhausted. """ + if _use_cli_backend(): + from core.llm_cli import call_cli + return call_cli( + messages, + tools=tools, + model=model, + max_completion_tokens=max_completion_tokens, + timeout=timeout, + max_retries=max_retries, + tool_choice=tool_choice, + ) + model_key = model.lower().strip() # Dispatch to Responses API for models that need it. gpt-5.5 goes here so @@ -549,6 +566,18 @@ def call_llm( For function-calling agent loops, use call_azure_openai() directly. """ + if backend == "cli" or (_use_cli_backend() and backend in ("azure", "gpt-5.4")): + from core.llm_cli import call_cli + messages = [] + if system: + messages.append({"role": "system", "content": system}) + messages.append({"role": "user", "content": prompt}) + try: + msg = call_cli(messages, **kw) + return msg.get("content", "") or "" + except LLMError as e: + log.error("CLI call_llm failed: %s", e) + return "" if backend == "azure" or backend == "gpt-5.4": messages = [] if system: diff --git a/core/llm_cli.py b/core/llm_cli.py new file mode 100644 index 00000000..a1589285 --- /dev/null +++ b/core/llm_cli.py @@ -0,0 +1,371 @@ +""" +core/llm_cli.py + +CLI LLM backend: drive a locally logged-in agent CLI (Claude Code) as the +model provider instead of API keys. Activated when ``R2S_LLM_BACKEND=cli``; +``core.llm.call_azure_openai`` dispatches here before touching Azure. + +Why not ``claude -p`` one-shot per call: the harness agent loop +(``core/agent_executor.py``) is multi-turn — it appends assistant messages +and tool results to a growing conversation. One-shot print mode loses all +prior context. This module instead maps each harness conversation to a +Claude Code session: + + - First call of a conversation → ``claude -p --output-format json`` + (system prompt via ``--system-prompt``), capture ``session_id``. + - Subsequent calls → ``claude -p --resume `` + with ONLY the newly appended messages serialized as the next user turn. + +Conversations are append-only (verified in ``agent_executor._run_loop``), +so a fingerprint of the first two messages identifies the conversation and +``sent_count`` tracks how much of it has been forwarded to the CLI. + +Function calling is emulated with a text protocol: tool JSON schemas are +injected into the system prompt and the model must reply with a strict +JSON envelope, which is parsed back into the OpenAI assistant-message +shape (``content`` + ``tool_calls``) so no call site changes are needed. + +Config: + R2S_LLM_BACKEND=cli enable this backend + R2S_CLI_TOOL=claude which CLI to use (only "claude" supported in v1) + R2S_CLI_MODEL=... optional model override passed to --model +""" +from __future__ import annotations + +import hashlib +import json +import logging +import os +import re +import shutil +import subprocess +import threading + +from core.llm import LLMError + +log = logging.getLogger("llm_cli") + +# --------------------------------------------------------------------------- +# Conversation/session registry +# --------------------------------------------------------------------------- + + +class _Session: + __slots__ = ("session_id", "sent_count") + + def __init__(self, session_id: str, sent_count: int): + self.session_id = session_id + self.sent_count = sent_count + + +_SESSIONS: dict[str, _Session] = {} +_SESSIONS_LOCK = threading.Lock() + + +def _fingerprint(messages: list[dict]) -> str: + """Identify a conversation by its immutable root (system + first user msg).""" + h = hashlib.sha256() + for msg in messages[:2]: + h.update(json.dumps(msg, sort_keys=True, default=str).encode()) + h.update(b"\x00") + return h.hexdigest() + + +def reset_sessions() -> None: + """Drop all session mappings (tests, long-running daemons between tasks).""" + with _SESSIONS_LOCK: + _SESSIONS.clear() + + +# --------------------------------------------------------------------------- +# Message serialization +# --------------------------------------------------------------------------- + + +def _content_to_text(content) -> str: + """Flatten OpenAI content (str or parts list) to plain text. + + Image parts cannot be attached through the CLI; they are replaced with an + explicit placeholder so the model knows material was omitted. + """ + if isinstance(content, str): + return content + if not isinstance(content, list): + return str(content or "") + texts: list[str] = [] + n_images = 0 + for part in content: + if not isinstance(part, dict): + continue + if part.get("type") == "text": + texts.append(part.get("text", "")) + elif part.get("type") == "image_url": + n_images += 1 + if n_images: + texts.append(f"[{n_images} image(s) omitted: CLI backend cannot attach images]") + return "\n".join(t for t in texts if t) + + +def _serialize_new_turns(new_messages: list[dict]) -> str: + """Serialize newly appended messages into one user-turn text. + + Assistant messages are skipped: they were produced by this backend from + the CLI's own replies, so the CLI session already contains them. + """ + blocks: list[str] = [] + for msg in new_messages: + role = msg.get("role") + if role == "assistant": + continue + if role == "tool": + name = msg.get("name") or msg.get("tool_call_id") or "unknown" + blocks.append( + f"[Tool result for `{name}` " + f"(call id {msg.get('tool_call_id', '?')})]:\n" + + _content_to_text(msg.get("content")) + ) + elif role in ("user", "system"): + text = _content_to_text(msg.get("content")) + if text: + blocks.append(text) + return "\n\n".join(blocks) + + +# --------------------------------------------------------------------------- +# Tool-call protocol (text emulation of OpenAI function calling) +# --------------------------------------------------------------------------- + +_TOOL_PROTOCOL = """\ +TOOL PROTOCOL (mandatory) +You are running inside an automated harness. You do NOT have direct tool \ +access; the harness executes tools for you. Available tools (JSON Schema): +{schemas} + +Reply with EXACTLY ONE JSON object and nothing else (no markdown fences, \ +no prose before or after). Two forms: +1. To call one or more tools: + {{"content": null, "tool_calls": [{{"id": "call_1", "name": "", "arguments": {{...}}}}]}} +2. When no tool is needed (final answer / status): + {{"content": "", "tool_calls": []}} +Every reply must be one of these two forms.""" + + +def _build_system_prompt(base: str, tools: list[dict] | None) -> str: + if not tools: + return base + schemas = json.dumps(tools, ensure_ascii=False, indent=1) + return (base + "\n\n" if base else "") + _TOOL_PROTOCOL.format(schemas=schemas) + + +def _extract_json_object(text: str) -> dict | None: + """Parse the first balanced {...} object from model output.""" + text = text.strip() + # Strip markdown fences if the whole reply is fenced. + fence = re.fullmatch(r"```(?:json)?\s*([\s\S]*?)\s*```", text) + if fence: + text = fence.group(1).strip() + try: + obj = json.loads(text) + return obj if isinstance(obj, dict) else None + except json.JSONDecodeError: + pass + # Balanced-brace scan for the first complete object. + start = text.find("{") + while start != -1: + depth = 0 + in_str = False + esc = False + for i in range(start, len(text)): + ch = text[i] + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + continue + if ch == '"': + in_str = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + try: + obj = json.loads(text[start : i + 1]) + return obj if isinstance(obj, dict) else None + except json.JSONDecodeError: + break + start = text.find("{", start + 1) + return None + + +def _envelope_to_message(env: dict) -> dict: + """Convert the parsed tool-protocol envelope to an OpenAI assistant msg.""" + msg: dict = {"role": "assistant", "content": env.get("content") or ""} + tool_calls = env.get("tool_calls") or [] + if tool_calls: + msg["tool_calls"] = [ + { + "id": str(tc.get("id") or f"call_{i}"), + "type": "function", + "function": { + "name": tc["name"], + "arguments": json.dumps(tc.get("arguments") or {}), + }, + } + for i, tc in enumerate(tool_calls) + if isinstance(tc, dict) and tc.get("name") + ] + if not msg["tool_calls"]: + del msg["tool_calls"] + return msg + + +# --------------------------------------------------------------------------- +# Claude CLI invocation +# --------------------------------------------------------------------------- + + +def _cli_tool() -> str: + return os.environ.get("R2S_CLI_TOOL", "claude").strip().lower() + + +def _run_claude( + prompt: str, + *, + system_prompt: str | None, + resume_id: str | None, + model: str | None, + timeout: int, +) -> dict: + """One claude -p invocation. Returns the parsed --output-format json dict.""" + exe = shutil.which("claude") + if not exe: + raise LLMError("R2S_CLI_TOOL=claude but `claude` is not on PATH") + cmd = [exe, "-p", "--output-format", "json"] + if system_prompt: + cmd += ["--system-prompt", system_prompt] + if resume_id: + cmd += ["--resume", resume_id] + if model: + cmd += ["--model", model] + log.debug("claude call (resume=%s, prompt=%d chars)", bool(resume_id), len(prompt)) + try: + proc = subprocess.run( + cmd, + input=prompt, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired as e: + raise LLMError(f"claude CLI timed out after {timeout}s") from e + if proc.returncode != 0: + raise LLMError(f"claude CLI exit {proc.returncode}: {proc.stderr[:500]}") + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError as e: + raise LLMError(f"claude CLI returned non-JSON output: {proc.stdout[:300]}") from e + if data.get("is_error"): + raise LLMError(f"claude CLI error: {data.get('result', '')[:500]}") + return data + + +# --------------------------------------------------------------------------- +# Public entry point (mirrors call_azure_openai's contract) +# --------------------------------------------------------------------------- + + +def call_cli( + messages: list[dict], + *, + tools: list[dict] | None = None, + model: str | None = None, + max_completion_tokens: int = 4096, # noqa: ARG001 - CLI has no token cap knob + timeout: int = 300, + max_retries: int = 3, + tool_choice=None, + **_: object, +) -> dict: + """Multi-turn LLM call via a local agent CLI. Drop-in for call_azure_openai. + + Returns an OpenAI-style assistant message dict (``content`` and optional + ``tool_calls``). Raises LLMError after exhausting retries. + """ + tool = _cli_tool() + if tool != "claude": + raise LLMError(f"R2S_CLI_TOOL={tool!r} not supported yet (v1: claude only)") + + cli_model = os.environ.get("R2S_CLI_MODEL") or None + timeout = max(60, timeout) + fp = _fingerprint(messages) + + with _SESSIONS_LOCK: + session = _SESSIONS.get(fp) + + last_error: Exception | None = None + for attempt in range(1, max_retries + 1): + try: + if session is None: + system_msg = next((m for m in messages if m.get("role") == "system"), None) + base_system = _content_to_text(system_msg.get("content")) if system_msg else "" + system_prompt = _build_system_prompt(base_system, tools) + first_turn = _serialize_new_turns( + [m for m in messages if m.get("role") != "system"] + ) + data = _run_claude( + first_turn, + system_prompt=system_prompt or None, + resume_id=None, + model=cli_model, + timeout=timeout, + ) + session = _Session(data["session_id"], len(messages)) + with _SESSIONS_LOCK: + _SESSIONS[fp] = session + else: + new_tail = messages[session.sent_count :] + turn_text = _serialize_new_turns(new_tail) + if tool_choice: + # Harness is forcing a tool call this round. + if isinstance(tool_choice, dict): + fn = tool_choice.get("function", {}).get("name", "?") + turn_text += ( + f"\n\n[HARNESS] You MUST call the tool `{fn}` now " + "(form 1 of the protocol)." + ) + else: + turn_text += ( + "\n\n[HARNESS] You MUST call at least one tool now " + "(form 1 of the protocol)." + ) + # The session already carries the system prompt from the first + # turn; overriding it on resume would drop domain instructions. + data = _run_claude( + turn_text or "Continue.", + system_prompt=None, + resume_id=session.session_id, + model=cli_model, + timeout=timeout, + ) + + result_text = data.get("result") or "" + if not tools: + session.sent_count = len(messages) + return {"role": "assistant", "content": result_text} + + env = _extract_json_object(result_text) + if env is None: + raise LLMError( + f"CLI reply is not a valid tool-protocol envelope: {result_text[:300]}" + ) + session.sent_count = len(messages) + return _envelope_to_message(env) + + except LLMError as e: + last_error = e + log.warning("CLI call attempt %d/%d failed: %s", attempt, max_retries, e) + + raise LLMError(f"CLI backend exhausted {max_retries} retries. Last error: {last_error}") diff --git a/skill/Resource2Skill/SKILL.md b/skill/Resource2Skill/SKILL.md new file mode 100644 index 00000000..4ce849ca --- /dev/null +++ b/skill/Resource2Skill/SKILL.md @@ -0,0 +1,124 @@ +--- +name: Resource2Skill +author: Resource2Skill project +description: Drive the Resource2Skill CLI from an agent session. No LLM API key is required if the target machine has claude/codex/kimi/omp installed and logged in. +--- + +# Resource2Skill + +Operate the Resource2Skill skill-distillation harness from within an agent session. + +## When to use + +- The user wants to turn a YouTube tutorial, article, repo, or reference artifact into an executable agent skill. +- The user wants to search, inspect, or execute skills in the local `skills_library/`. +- The user wants to run the agent loop to build a web page, PowerPoint deck, Excel workbook, Blender scene, or REAPER-style audio project using the skill library. + +## Prerequisites + +- Python 3.11+ and the project dependencies installed: + ```bash + cd /path/to/Resource2Skill + pip install -r requirements.txt + ``` +- `yt-dlp` and `ffmpeg` on PATH for the video-analysis path. +- At least one local agent CLI installed and authenticated (Claude Code is best supported): + - `claude` (Claude Code) + - `codex` (OpenAI Codex CLI) + - `kimi` (Moonshot Kimi Code CLI) + - `omp` (Oh My Pi agent harness) + +## Configuration + +Set the backend once per shell or task: + +```bash +# Use a local agent CLI instead of API keys +export R2S_LLM_BACKEND=cli +export R2S_CLI_TOOL=claude # claude | codex | kimi | omp + +# Optional: use subtitle+keyframes for video analysis (no Gemini key) +export R2S_VIDEO_BACKEND=cli +``` + +If you leave these unset, Resource2Skill falls back to its original behavior: +`AZURE_OPENAI_API_KEY` for agent reasoning and `GEMINI_API_KEY` for video analysis. + +## Commands + +All commands run from the repository root. + +### Discover available domains + +```bash +python cli.py domains +``` + +Use this to list the supported authoring domains (web, ppt, excel, blender, reaper). + +### Distill a skill from a YouTube video + +```bash +python cli.py analyze --domain web \ + --video "https://www.youtube.com/watch?v=yefgBA1CecI" \ + -o /tmp/typewriter_skill.md +``` + +The command writes a Markdown skill analysis. With `R2S_VIDEO_BACKEND=cli` it uses +yt-dlp subtitles + ffmpeg keyframes instead of Gemini video understanding. + +### Run the agent loop to execute a task + +```bash +python cli.py agent --domain web \ + --task "Build a responsive glassmorphism navigation bar" \ + --dry-run +``` + +`--dry-run` uses a mock MCP server so the agent plans and shows tool calls without +muting real files. Remove `--dry-run` to let the agent write files through the domain MCP server. + +### Retrieve skills for a query + +```bash +python cli.py retrieve --domain web --query "typewriter animation" --select 3 +``` + +Returns the top-3 matching skills from `skills_library/web/`. + +### Validate a domain configuration + +```bash +python cli.py validate-domain --domain web +``` + +Run this after changing a domain YAML or before a long agent loop to catch config errors early. + +## Output layout + +- `skills_wiki//` — structured, searchable skill entries. +- `skills_library//` — executable code assets used by the agent loop. +- Results written with `-o` go to the path you specify. + +## Key limitations + +- `R2S_VIDEO_BACKEND=cli` currently supports image ingestion best with `claude`, because claude can read local image files in headless mode. kimi, codex, and omp either lack image input or have not been verified. +- The CLI LLM backend currently implements `claude` first; codex/kimi/omp backends are planned. +- Embeddings (`python cli.py build`) still require a Gemini key unless you rely on keyword-only retrieval. + +## Examples + +```bash +# 1. Keyless skill distillation from a short web tutorial +export R2S_LLM_BACKEND=cli +export R2S_CLI_TOOL=claude +export R2S_VIDEO_BACKEND=cli +python cli.py analyze --domain web \ + --video "https://www.youtube.com/watch?v=yefgBA1CecI" \ + -o /tmp/typewriter_skill.md + +# 2. Keyless agent loop (dry-run) for a web component +python cli.py agent --domain web \ + --task "Create a pure-CSS typewriter hero text effect" \ + --dry-run +``` From 5111e961fa8aeb20fe3d2bf78e5c016d38a5b1bf Mon Sep 17 00:00:00 2001 From: dull bird <1155115927@link.cuhk.edu.hk> Date: Sat, 25 Jul 2026 01:07:55 +0800 Subject: [PATCH 2/4] feat: auto-fallback to CLI backend when API keys are missing - Detect locally installed agent CLIs (claude, codex, kimi, omp) and default R2S_LLM_BACKEND=cli when AZURE_OPENAI_API_KEY is absent. - Default R2S_VIDEO_BACKEND=cli when GEMINI_API_KEY is absent so video analysis uses yt-dlp subtitles + ffmpeg keyframes without configuration. - Update skill/Resource2Skill/SKILL.md with discovery/retrieval instructions and explain the skill.json format. - Update README no-key section with the new auto-fallback behavior. --- README.md | 4 ++ core/analyzer.py | 10 ++++- core/llm.py | 53 ++++++++++++++++++++++++- skill/Resource2Skill/SKILL.md | 75 +++++++++++++++++++++++++++++++++-- 4 files changed, 137 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 18aa53f5..53a7fd45 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,10 @@ export R2S_CLI_TOOL=claude export R2S_VIDEO_BACKEND=cli ``` +You can also leave the variables unset: if `AZURE_OPENAI_API_KEY` is missing and +an agent CLI is on PATH, the system defaults to `R2S_LLM_BACKEND=cli`. If +`GEMINI_API_KEY` is missing, video analysis defaults to `R2S_VIDEO_BACKEND=cli`. + Tested: `claude` supports multi-turn session resume, a text-based tool-call protocol, and reading local image files in headless mode. `codex`, `kimi`, and `omp` backends are not yet implemented. diff --git a/core/analyzer.py b/core/analyzer.py index 58fa706d..c0511fdb 100644 --- a/core/analyzer.py +++ b/core/analyzer.py @@ -91,7 +91,15 @@ def analyze_video( Returns: Raw Markdown analysis from the model. """ - if os.environ.get("R2S_VIDEO_BACKEND", "").strip().lower() == "cli": + video_backend = os.environ.get("R2S_VIDEO_BACKEND", "").strip().lower() + gemini_key = os.environ.get("GEMINI_API_KEY") + if video_backend == "cli" or (not video_backend and not gemini_key): + if not video_backend and not gemini_key: + import shutil as _shutil + if _shutil.which("claude") or _shutil.which("yt-dlp"): + log.info("GEMINI_API_KEY not set; defaulting to CLI video backend") + else: + log.warning("GEMINI_API_KEY not set and no CLI/yt-dlp found; CLI video backend may fail") return analyze_video_cli( video_url, prompt_path=prompt_path, diff --git a/core/llm.py b/core/llm.py index 8a175996..1309091c 100644 --- a/core/llm.py +++ b/core/llm.py @@ -16,6 +16,7 @@ import json import logging import os +import shutil import threading import time from typing import Any @@ -188,9 +189,42 @@ class LLMError(Exception): """Raised when all LLM call attempts fail.""" +def _cli_tool_available(preferred: str | None = None) -> str | None: + """Return the first locally installed agent CLI we can find.""" + candidates = [preferred] if preferred else ["claude", "codex", "kimi", "omp"] + candidates = [c for c in candidates if c] + for tool in candidates: + if shutil.which(tool): + return tool + return None + + +def _auto_configure_cli_backend() -> bool: + """Enable CLI backend when no API keys are present and a CLI is installed. + + This makes keyless operation work out of the box: if the user has not + configured R2S_LLM_BACKEND and has no AZURE_OPENAI_API_KEY, we default to + the locally authenticated agent CLI (Claude Code first, then others). + """ + configured = os.environ.get("R2S_LLM_BACKEND", "").strip().lower() + if configured == "cli": + return True + if configured and configured != "cli": + return False + if os.environ.get("AZURE_OPENAI_API_KEY"): + return False + tool = _cli_tool_available() + if tool: + os.environ.setdefault("R2S_LLM_BACKEND", "cli") + os.environ.setdefault("R2S_CLI_TOOL", tool) + log.info("No AZURE_OPENAI_API_KEY found; defaulting to CLI backend (%s)", tool) + return True + return False + + def _use_cli_backend() -> bool: """Route LLM calls through a local agent CLI (no API keys) when set.""" - return os.environ.get("R2S_LLM_BACKEND", "").strip().lower() == "cli" + return _auto_configure_cli_backend() def _chat_to_responses_input(messages: list[dict]) -> list[dict]: @@ -566,6 +600,8 @@ def call_llm( For function-calling agent loops, use call_azure_openai() directly. """ + # Keyless fallback: if the requested backend has no API key but a local + # agent CLI is available, route through it. if backend == "cli" or (_use_cli_backend() and backend in ("azure", "gpt-5.4")): from core.llm_cli import call_cli messages = [] @@ -578,6 +614,21 @@ def call_llm( except LLMError as e: log.error("CLI call_llm failed: %s", e) return "" + if backend == "gemini" and not os.environ.get("GEMINI_API_KEY") and _cli_tool_available(): + log.info("GEMINI_API_KEY not set; falling back to CLI backend for call_llm") + os.environ.setdefault("R2S_LLM_BACKEND", "cli") + os.environ.setdefault("R2S_CLI_TOOL", _cli_tool_available()) + from core.llm_cli import call_cli + messages = [] + if system: + messages.append({"role": "system", "content": system}) + messages.append({"role": "user", "content": prompt}) + try: + msg = call_cli(messages, **kw) + return msg.get("content", "") or "" + except LLMError as e: + log.error("CLI call_llm failed: %s", e) + return "" if backend == "azure" or backend == "gpt-5.4": messages = [] if system: diff --git a/skill/Resource2Skill/SKILL.md b/skill/Resource2Skill/SKILL.md index 4ce849ca..5dae1fe4 100644 --- a/skill/Resource2Skill/SKILL.md +++ b/skill/Resource2Skill/SKILL.md @@ -30,7 +30,17 @@ Operate the Resource2Skill skill-distillation harness from within an agent sessi ## Configuration -Set the backend once per shell or task: +The fastest way is to create a `.env` file in the repo root: + +```bash +cp .env.example .env +# edit .env and uncomment the CLI backend lines +``` + +`.env` is **not** automatically read by the OS; `cli.py` uses `python-dotenv` to +load it when you run a command. + +You can also export per shell: ```bash # Use a local agent CLI instead of API keys @@ -41,8 +51,21 @@ export R2S_CLI_TOOL=claude # claude | codex | kimi | omp export R2S_VIDEO_BACKEND=cli ``` -If you leave these unset, Resource2Skill falls back to its original behavior: -`AZURE_OPENAI_API_KEY` for agent reasoning and `GEMINI_API_KEY` for video analysis. +### Default behavior when no keys are set + +If you leave the variables unset, the system will **auto-detect** a keyless path +whenever possible: + +- If `AZURE_OPENAI_API_KEY` is missing and an agent CLI (`claude`, `codex`, + `kimi`, `omp`) is on PATH, `R2S_LLM_BACKEND` defaults to `cli`. +- If `GEMINI_API_KEY` is missing, `R2S_VIDEO_BACKEND` defaults to `cli` + (subtitle + keyframes). + +So on a machine with `claude` and `yt-dlp` installed, you can often run without +setting anything manually. + +If you explicitly set API keys, the CLI backend is ignored unless you also set +`R2S_LLM_BACKEND=cli`. ## Commands @@ -94,6 +117,52 @@ python cli.py validate-domain --domain web Run this after changing a domain YAML or before a long agent loop to catch config errors early. +## How skills are stored and discovered + +Resource2Skill keeps skills in two roots: + +- `skills_wiki//` — browse/search entries. Each skill has + `code/skill.json` (metadata + code pointers) and `text/overview.md` (human + readable summary). +- `skills_library//` — executable assets. Each skill is a folder + containing `skill.json` plus any code, frames, or helper files. + +The `skill.json` file is **not** a Markdown skill file. It is structured JSON +with fields such as `skill_id`, `skill_name`, `domain`, `source` (provenance), +`analysis` (the Markdown distillation), and executable code. Use the CLI or read +the JSON directly. + +### Search for a skill + +```bash +python cli.py retrieve --domain web --query "typewriter animation" --select 3 +``` + +This returns the top matching skills from `skills_library/web/`. The query can +use natural language; keyword fallback works even without embedding keys. + +### Inspect a specific skill + +Option 1 — read the JSON directly: + +```bash +cat skills_library/web/animation/dynamic_pure_css_typewriter_effect_12449511/skill.json +``` + +Option 2 — use the agent loop dry-run to see how the skill would be executed: + +```bash +python cli.py execute --domain web \ + --skill dynamic_pure_css_typewriter_effect_12449511 \ + --dry-run +``` + +### List skills in a domain + +```bash +find skills_library/web -name skill.json | head -20 +``` + ## Output layout - `skills_wiki//` — structured, searchable skill entries. From 4a3680274af0e04fe2ec5e5361631827374a78a0 Mon Sep 17 00:00:00 2001 From: dull bird <1155115927@link.cuhk.edu.hk> Date: Sat, 25 Jul 2026 01:57:01 +0800 Subject: [PATCH 3/4] feat: local ASR fallback when native video subtitles are missing - Add _download_audio, _run_asr, and helpers to core/analyzer_cli_video.py. - Support R2S_VIDEO_ASR=auto|faster-whisper|whisper and R2S_VIDEO_ASR_MODEL. - Fallback only triggers after yt-dlp reports no native VTT subtitles. - Benchmark: faster-whisper tiny is ~5x faster than openai-whisper tiny on a 60s clip (3.9s vs 20.6s) at comparable quality. - Update README, .env.example, and SKILL.md with the new env variables and limitations. --- .env.example | 7 ++ README.md | 5 + core/analyzer_cli_video.py | 176 +++++++++++++++++++++++++++++++++- skill/Resource2Skill/SKILL.md | 14 ++- 4 files changed, 193 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index 76c55980..83757ea2 100644 --- a/.env.example +++ b/.env.example @@ -35,3 +35,10 @@ # Set R2S_VIDEO_BACKEND=cli to analyze YouTube videos using local yt-dlp # subtitles + ffmpeg keyframes instead of Gemini native video understanding. # R2S_VIDEO_BACKEND=cli + +# When native subtitles are missing, fall back to local ASR on the downloaded +# audio. Requires faster-whisper or openai-whisper installed. Set to "auto" +# to use the fastest available backend, or explicitly "faster-whisper" / "whisper". +# R2S_VIDEO_ASR=auto +# R2S_VIDEO_ASR_MODEL=tiny # tiny | base | small (faster-whisper also supports medium/large-v3) + diff --git a/README.md b/README.md index 53a7fd45..c7c58d23 100644 --- a/README.md +++ b/README.md @@ -72,11 +72,16 @@ export R2S_CLI_TOOL=claude # video analysis also works without Gemini by using yt-dlp subtitles + ffmpeg keyframes export R2S_VIDEO_BACKEND=cli + +# when no native subtitles exist, fall back to local ASR (faster-whisper or openai-whisper) +export R2S_VIDEO_ASR=auto ``` You can also leave the variables unset: if `AZURE_OPENAI_API_KEY` is missing and an agent CLI is on PATH, the system defaults to `R2S_LLM_BACKEND=cli`. If `GEMINI_API_KEY` is missing, video analysis defaults to `R2S_VIDEO_BACKEND=cli`. +If `R2S_VIDEO_ASR=auto` is set and faster-whisper/openai-whisper is installed, +native-subtitle failure falls back to local ASR on the downloaded audio. Tested: `claude` supports multi-turn session resume, a text-based tool-call protocol, and reading local image files in headless mode. `codex`, `kimi`, and diff --git a/core/analyzer_cli_video.py b/core/analyzer_cli_video.py index 31b2ffa5..68425f95 100644 --- a/core/analyzer_cli_video.py +++ b/core/analyzer_cli_video.py @@ -85,10 +85,16 @@ def _fetch_vtt(video_url: str, output_dir: Path, *, start_offset: str | None, en # yt-dlp names files like video.en.auto.vtt or video.zh-Hans.vtt candidates = sorted(glob.glob(str(base) + "*.vtt")) - if not candidates: - log.warning("No VTT subtitles found for %s", video_url) - return None - return Path(candidates[0]) + if candidates: + return Path(candidates[0]) + + # No native subtitles. Try local ASR if the user enabled it. + asr_path = _run_asr(video_url, output_dir, start_offset=start_offset, end_offset=end_offset) + if asr_path and asr_path.exists(): + return asr_path + + log.warning("No VTT subtitles or ASR output found for %s", video_url) + return None def _parse_vtt(vtt_path: Path) -> str: @@ -197,6 +203,168 @@ def _fmt_time(seconds: int) -> str: return f"{m}:{s:02d}" +# --------------------------------------------------------------------------- +# ASR fallback when no native subtitles exist +# --------------------------------------------------------------------------- + + +def _asr_backend() -> str | None: + """Return the configured ASR backend, or None if ASR is disabled. + + Env: + R2S_VIDEO_ASR=auto|faster-whisper|whisper|none + R2S_VIDEO_ASR_MODEL=tiny|base|small (default: tiny) + """ + backend = os.environ.get("R2S_VIDEO_ASR", "auto").strip().lower() + if backend in ("", "none", "0", "false"): + return None + if backend != "auto": + return backend + # auto: prefer faster-whisper if importable, else openai-whisper + try: + import faster_whisper # noqa: F401 + return "faster-whisper" + except ImportError: + pass + try: + import whisper # noqa: F401 + return "whisper" + except ImportError: + return None + + +def _asr_model() -> str: + return os.environ.get("R2S_VIDEO_ASR_MODEL", "tiny").strip().lower() or "tiny" + + +def _download_audio( + video_url: str, + output_path: Path, + *, + start_offset: str | None, + end_offset: str | None, + timeout: int = 180, +) -> bool: + """Download/extract audio from a video URL to a WAV file.""" + cmd = [ + "yt-dlp", + "-x", "--audio-format", "wav", "--audio-quality", "0", + "--no-playlist", "--quiet", + "-o", str(output_path.with_suffix(".%(ext)s")), + ] + if start_offset or end_offset: + def _fmt_sec(offset: str) -> str: + sec = _parse_offset(offset) if isinstance(offset, str) and offset else 0 + h, rem = divmod(sec, 3600) + m, s = divmod(rem, 60) + return f"{h:02d}:{m:02d}:{s:02d}" + start = _fmt_sec(start_offset) if start_offset else "0:00:00" + end = _fmt_sec(end_offset) if end_offset else "9999:59:59" + cmd += ["--download-sections", f"*{start}-{end}"] + cmd.append(video_url) + try: + subprocess.run(cmd, check=True, timeout=timeout, capture_output=True) + return output_path.exists() and output_path.stat().st_size > 0 + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + log.warning("Failed to download audio: %s", e) + return False + + +def _segments_to_vtt(segments, output_path: Path) -> Path: + """Write faster-whisper / whisper segments to a WebVTT file.""" + def _seg_val(seg, name: str): + if hasattr(seg, name): + return getattr(seg, name) + if isinstance(seg, dict): + return seg.get(name, 0 if name in ("start", "end") else "") + return 0 if name in ("start", "end") else "" + + lines = ["WEBVTT", ""] + for seg in segments: + start = _seg_val(seg, "start") + end = _seg_val(seg, "end") + text = str(_seg_val(seg, "text")).strip() + if not text: + continue + lines.append(f"{_vtt_timestamp(float(start))} --> {_vtt_timestamp(float(end))}") + lines.append(text) + lines.append("") + output_path.write_text("\n".join(lines), encoding="utf-8") + return output_path + + +def _vtt_timestamp(seconds: float) -> str: + """Format seconds as HH:MM:SS.mmm for VTT.""" + ms = int((seconds % 1) * 1000) + s = int(seconds) % 60 + m = (int(seconds) // 60) % 60 + h = int(seconds) // 3600 + return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}" + + +def _transcribe_audio_faster_whisper(audio_path: Path, output_path: Path, model: str) -> Path | None: + """Transcribe audio with faster-whisper and write a VTT file.""" + try: + from faster_whisper import WhisperModel + except ImportError as e: + log.warning("faster-whisper not installed: %s", e) + return None + log.info("Running faster-whisper (%s) on %s", model, audio_path) + try: + # int8 on CPU keeps memory low; Apple Silicon also handles this fine. + m = WhisperModel(model, device="cpu", compute_type="int8") + segments, _ = m.transcribe(str(audio_path), beam_size=5, word_timestamps=False) + return _segments_to_vtt(segments, output_path) + except Exception as e: + log.warning("faster-whisper transcription failed: %s", e) + return None + + +def _transcribe_audio_whisper(audio_path: Path, output_path: Path, model: str) -> Path | None: + """Transcribe audio with openai-whisper and write a VTT file.""" + try: + import whisper + except ImportError as e: + log.warning("openai-whisper not installed: %s", e) + return None + log.info("Running openai-whisper (%s) on %s", model, audio_path) + try: + m = whisper.load_model(model) + result = m.transcribe(str(audio_path), verbose=False) + return _segments_to_vtt(result.get("segments", []), output_path) + except Exception as e: + log.warning("openai-whisper transcription failed: %s", e) + return None + + +def _run_asr( + video_url: str, + output_dir: Path, + *, + start_offset: str | None, + end_offset: str | None, +) -> Path | None: + """Generate a VTT file via local ASR when native subtitles are missing.""" + backend = _asr_backend() + if not backend: + return None + model = _asr_model() + audio_path = output_dir / "audio.wav" + if not _download_audio(video_url, audio_path, start_offset=start_offset, end_offset=end_offset): + return None + vtt_path = output_dir / "asr.vtt" + if backend == "faster-whisper": + result = _transcribe_audio_faster_whisper(audio_path, vtt_path, model) + elif backend == "whisper": + result = _transcribe_audio_whisper(audio_path, vtt_path, model) + else: + log.warning("Unknown ASR backend: %s", backend) + return None + if result and result.exists(): + log.info("ASR generated subtitle: %s", result) + return result + + # --------------------------------------------------------------------------- # Frame extraction helper # --------------------------------------------------------------------------- diff --git a/skill/Resource2Skill/SKILL.md b/skill/Resource2Skill/SKILL.md index 5dae1fe4..c3b8cdf7 100644 --- a/skill/Resource2Skill/SKILL.md +++ b/skill/Resource2Skill/SKILL.md @@ -49,6 +49,10 @@ export R2S_CLI_TOOL=claude # claude | codex | kimi | omp # Optional: use subtitle+keyframes for video analysis (no Gemini key) export R2S_VIDEO_BACKEND=cli + +# Optional: if native subtitles are missing, generate them with local ASR +export R2S_VIDEO_ASR=auto # auto | faster-whisper | whisper +export R2S_VIDEO_ASR_MODEL=tiny # tiny | base | small ``` ### Default behavior when no keys are set @@ -60,12 +64,11 @@ whenever possible: `kimi`, `omp`) is on PATH, `R2S_LLM_BACKEND` defaults to `cli`. - If `GEMINI_API_KEY` is missing, `R2S_VIDEO_BACKEND` defaults to `cli` (subtitle + keyframes). +- If `R2S_VIDEO_ASR=auto` is set and `faster-whisper` or `openai-whisper` is + installed, missing native subtitles fall back to local ASR on the audio track. -So on a machine with `claude` and `yt-dlp` installed, you can often run without -setting anything manually. - -If you explicitly set API keys, the CLI backend is ignored unless you also set -`R2S_LLM_BACKEND=cli`. +So on a machine with `claude`, `yt-dlp`, and `faster-whisper` installed, you can +often run without setting anything manually. ## Commands @@ -174,6 +177,7 @@ find skills_library/web -name skill.json | head -20 - `R2S_VIDEO_BACKEND=cli` currently supports image ingestion best with `claude`, because claude can read local image files in headless mode. kimi, codex, and omp either lack image input or have not been verified. - The CLI LLM backend currently implements `claude` first; codex/kimi/omp backends are planned. - Embeddings (`python cli.py build`) still require a Gemini key unless you rely on keyword-only retrieval. +- ASR fallback (`R2S_VIDEO_ASR`) requires `faster-whisper` or `openai-whisper` and first model download is ~40MB (`tiny`) to ~150MB (`base`). Transcription speed depends on CPU; Apple Silicon with `faster-whisper` int8 is usable for short videos. ## Examples From c9a9f77a580bfdaf8070c3053d778f2201230b6a Mon Sep 17 00:00:00 2001 From: dull-bird Date: Mon, 27 Jul 2026 13:07:03 +0800 Subject: [PATCH 4/4] fix(cli): isolate sessions and honor explicit api keys --- core/agent_executor.py | 3 ++ core/analyzer.py | 2 +- core/llm.py | 10 +++-- core/llm_cli.py | 26 +++++++------ skill/Resource2Skill/SKILL.md | 3 +- test_keyless_cli_backend.py | 72 +++++++++++++++++++++++++++++++++++ 6 files changed, 99 insertions(+), 17 deletions(-) create mode 100644 test_keyless_cli_backend.py diff --git a/core/agent_executor.py b/core/agent_executor.py index 7168a42e..7d5ebefa 100644 --- a/core/agent_executor.py +++ b/core/agent_executor.py @@ -23,6 +23,7 @@ import logging import os import re +import uuid from collections import deque from dataclasses import dataclass, field from pathlib import Path @@ -2086,6 +2087,7 @@ async def _run_loop( {"role": "system", "content": system_prompt}, {"role": "user", "content": f"Task: {task}\n\n{initial_prompt}"}, ] + conversation_id = uuid.uuid4().hex # Inject reference frames as a vision message if available all_frames = [] @@ -2177,6 +2179,7 @@ async def _run_loop( messages, tools=openai_tools, model=self._model, + conversation_id=conversation_id, reasoning_effort=self._reasoning, max_completion_tokens=16384, max_retries=10, diff --git a/core/analyzer.py b/core/analyzer.py index c0511fdb..870a4382 100644 --- a/core/analyzer.py +++ b/core/analyzer.py @@ -92,7 +92,7 @@ def analyze_video( Raw Markdown analysis from the model. """ video_backend = os.environ.get("R2S_VIDEO_BACKEND", "").strip().lower() - gemini_key = os.environ.get("GEMINI_API_KEY") + gemini_key = api_key or os.environ.get("GEMINI_API_KEY") if video_backend == "cli" or (not video_backend and not gemini_key): if not video_backend and not gemini_key: import shutil as _shutil diff --git a/core/llm.py b/core/llm.py index 1309091c..4ec6bc35 100644 --- a/core/llm.py +++ b/core/llm.py @@ -446,6 +446,7 @@ def call_azure_openai( *, tools: list[dict] | None = None, model: str = "gpt-5.4", + conversation_id: str | None = None, reasoning_effort: str = "medium", max_completion_tokens: int = 4096, timeout: int = 120, @@ -484,6 +485,7 @@ def call_azure_openai( messages, tools=tools, model=model, + conversation_id=conversation_id, max_completion_tokens=max_completion_tokens, timeout=timeout, max_retries=max_retries, @@ -593,6 +595,7 @@ def call_llm( system: str = "", *, backend: str = "gemini", + conversation_id: str | None = None, **kw, ) -> str: """ @@ -602,6 +605,7 @@ def call_llm( """ # Keyless fallback: if the requested backend has no API key but a local # agent CLI is available, route through it. + gemini_key = kw.get("api_key") or os.environ.get("GEMINI_API_KEY") if backend == "cli" or (_use_cli_backend() and backend in ("azure", "gpt-5.4")): from core.llm_cli import call_cli messages = [] @@ -609,12 +613,12 @@ def call_llm( messages.append({"role": "system", "content": system}) messages.append({"role": "user", "content": prompt}) try: - msg = call_cli(messages, **kw) + msg = call_cli(messages, conversation_id=conversation_id, **kw) return msg.get("content", "") or "" except LLMError as e: log.error("CLI call_llm failed: %s", e) return "" - if backend == "gemini" and not os.environ.get("GEMINI_API_KEY") and _cli_tool_available(): + if backend == "gemini" and not gemini_key and _cli_tool_available(): log.info("GEMINI_API_KEY not set; falling back to CLI backend for call_llm") os.environ.setdefault("R2S_LLM_BACKEND", "cli") os.environ.setdefault("R2S_CLI_TOOL", _cli_tool_available()) @@ -624,7 +628,7 @@ def call_llm( messages.append({"role": "system", "content": system}) messages.append({"role": "user", "content": prompt}) try: - msg = call_cli(messages, **kw) + msg = call_cli(messages, conversation_id=conversation_id, **kw) return msg.get("content", "") or "" except LLMError as e: log.error("CLI call_llm failed: %s", e) diff --git a/core/llm_cli.py b/core/llm_cli.py index a1589285..20047954 100644 --- a/core/llm_cli.py +++ b/core/llm_cli.py @@ -17,8 +17,9 @@ with ONLY the newly appended messages serialized as the next user turn. Conversations are append-only (verified in ``agent_executor._run_loop``), -so a fingerprint of the first two messages identifies the conversation and -``sent_count`` tracks how much of it has been forwarded to the CLI. +so each run uses an explicit conversation id when available, otherwise the +live messages list identity, and ``sent_count`` tracks how much of it has +been forwarded to the CLI. Function calling is emulated with a text protocol: tool JSON schemas are injected into the system prompt and the model must reply with a strict @@ -32,7 +33,6 @@ """ from __future__ import annotations -import hashlib import json import logging import os @@ -62,13 +62,16 @@ def __init__(self, session_id: str, sent_count: int): _SESSIONS_LOCK = threading.Lock() -def _fingerprint(messages: list[dict]) -> str: - """Identify a conversation by its immutable root (system + first user msg).""" - h = hashlib.sha256() - for msg in messages[:2]: - h.update(json.dumps(msg, sort_keys=True, default=str).encode()) - h.update(b"\x00") - return h.hexdigest() +def _conversation_key(messages: list[dict], conversation_id: str | None = None) -> str: + """Identify a conversation run. + + Prefer an explicit run id when the caller has one. Otherwise fall back to + the live messages list identity so identical prompts from separate runs do + not share a Claude session. + """ + if conversation_id: + return str(conversation_id) + return f"messages:{id(messages)}" def reset_sessions() -> None: @@ -283,6 +286,7 @@ def call_cli( *, tools: list[dict] | None = None, model: str | None = None, + conversation_id: str | None = None, max_completion_tokens: int = 4096, # noqa: ARG001 - CLI has no token cap knob timeout: int = 300, max_retries: int = 3, @@ -300,7 +304,7 @@ def call_cli( cli_model = os.environ.get("R2S_CLI_MODEL") or None timeout = max(60, timeout) - fp = _fingerprint(messages) + fp = _conversation_key(messages, conversation_id) with _SESSIONS_LOCK: session = _SESSIONS.get(fp) diff --git a/skill/Resource2Skill/SKILL.md b/skill/Resource2Skill/SKILL.md index c3b8cdf7..903ec396 100644 --- a/skill/Resource2Skill/SKILL.md +++ b/skill/Resource2Skill/SKILL.md @@ -1,6 +1,5 @@ --- -name: Resource2Skill -author: Resource2Skill project +name: resource2skill description: Drive the Resource2Skill CLI from an agent session. No LLM API key is required if the target machine has claude/codex/kimi/omp installed and logged in. --- diff --git a/test_keyless_cli_backend.py b/test_keyless_cli_backend.py new file mode 100644 index 00000000..a380d0d1 --- /dev/null +++ b/test_keyless_cli_backend.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from core import analyzer, llm, llm_cli + + +def test_call_cli_keeps_runs_separate_with_explicit_conversation_ids(monkeypatch): + llm_cli.reset_sessions() + resume_ids: list[str | None] = [] + + def fake_run_claude(prompt, *, system_prompt, resume_id, model, timeout): + resume_ids.append(resume_id) + return {"session_id": f"sess-{len(resume_ids)}", "result": "ok"} + + monkeypatch.setattr(llm_cli, "_run_claude", fake_run_claude) + + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "task"}, + ] + + llm_cli.call_cli(messages, conversation_id="run-a") + llm_cli.call_cli(messages, conversation_id="run-b") + + assert resume_ids == [None, None] + + +def test_call_llm_honors_explicit_gemini_api_key(monkeypatch): + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.delenv("R2S_LLM_BACKEND", raising=False) + monkeypatch.setattr(llm, "_cli_tool_available", lambda preferred=None: "claude") + monkeypatch.setattr(llm, "_call_gemini", lambda prompt, system="", **kw: "gemini-ok") + monkeypatch.setattr( + "core.llm_cli.call_cli", + lambda *args, **kwargs: pytest.fail("CLI fallback should not run when api_key is provided"), + ) + + assert llm.call_llm("prompt", backend="gemini", api_key="explicit-key") == "gemini-ok" + + +def test_analyze_video_honors_explicit_gemini_api_key(monkeypatch): + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + monkeypatch.delenv("R2S_VIDEO_BACKEND", raising=False) + monkeypatch.setattr( + analyzer, + "analyze_video_cli", + lambda *args, **kwargs: pytest.fail("CLI fallback should not run when api_key is provided"), + ) + monkeypatch.setattr(analyzer, "_load_prompt", lambda path=None: "prompt") + monkeypatch.setattr(analyzer, "_make_client", lambda api_key=None: object()) + monkeypatch.setattr( + analyzer, + "types", + SimpleNamespace( + FileData=lambda file_uri: SimpleNamespace(file_uri=file_uri), + VideoMetadata=lambda start_offset=None, end_offset=None: SimpleNamespace( + start_offset=start_offset, + end_offset=end_offset, + ), + Part=lambda **kwargs: SimpleNamespace(**kwargs), + ), + ) + monkeypatch.setattr( + analyzer, + "_call_with_retry", + lambda client, model, parts, max_retries=3: SimpleNamespace(text="gemini-ok"), + ) + + assert analyzer.analyze_video("https://example.com/video", api_key="explicit-key") == "gemini-ok"