diff --git a/coverage-ratchet.md b/coverage-ratchet.md index 2449c94..0b75540 100644 --- a/coverage-ratchet.md +++ b/coverage-ratchet.md @@ -2,12 +2,12 @@ Whole package (`podtx`) on this PR branch — **not** Codecov patch / diff coverage. -`statements=91.37% (min 65%) | branches=86.08% (min 45%) | combined=89.90% (informational)` +`statements=92.09% (min 65%) | branches=87.18% (min 45%) | combined=90.74% (informational)` | Metric | Value | Role | |--------|------:|------| -| Statements | 91.37% | Gated (min 65%) | -| Branches | 86.08% | Gated (min 45%) | -| Combined | 89.90% | Informational | +| Statements | 92.09% | Gated (min 65%) | +| Branches | 87.18% | Gated (min 45%) | +| Combined | 90.74% | Informational | **Status:** passed diff --git a/src/podtx/cli.py b/src/podtx/cli.py index f756a13..39220b7 100644 --- a/src/podtx/cli.py +++ b/src/podtx/cli.py @@ -21,17 +21,29 @@ from podtx.format_cmd import ( TranscriptJsonError, discover_transcript_jsons, + load_transcript_json, reformat_many, reformat_transcript, ) from podtx.nuggets import ( + DryRunEstimate, NuggetsError, _checked_formats, _valid_backend, + estimate_dry_run, extract_nuggets_transcript, nuggets_many, ) -from podtx.providers import ProviderError +from podtx.providers import ( + CatalogError, + ModelInfo, + ProviderError, + available_providers, + catalog_providers, + get_model, + list_models, + load_catalog, +) from podtx.rename_cmd import rename_many_from_title from podtx.summarize import SummarizeError, summarize_many, summarize_transcript from podtx.rss import FeedParseError, parse_feed, suggest_unique_slug @@ -1251,6 +1263,11 @@ def nuggets_cmd( temperature: Optional[float] = typer.Option(None, "--temperature", help="LLM temperature (default 0.3)"), max_input_chars: Optional[int] = typer.Option(None, "--max-input-chars", help="Chunk transcript at N chars, split on segment boundaries with overlap (default: 100000)"), force: bool = typer.Option(False, "--force", help="Re-extract even when a fresh sidecar exists"), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Estimate tokens and cost from the models.dev catalog without calling any backend or writing files", + ), quiet: bool = typer.Option(False, "--quiet", "-q"), ) -> None: """Extract durable insights ("nuggets") from existing transcript JSON. @@ -1314,6 +1331,21 @@ def nuggets_cmd( account=settings.nuggets_api_key_account, ) + if dry_run: + _run_nuggets_dry_run( + json_path=json_path, + feed=feed, + all_feeds=all_feeds, + limit=limit, + transcripts_root=settings.transcripts_dir(), + data_dir=settings.data_dir, + backend=effective_backend, + model=resolved_model, + max_input_chars=resolved_max, + quiet=quiet, + ) + return + if json_path is not None: path = json_path.expanduser() if not path.is_file(): @@ -1380,6 +1412,264 @@ def nuggets_cmd( raise typer.Exit(1) +def _fmt_integer(value: int | None) -> str: + return f"{value:,}" if value is not None else "—" + + +def _fmt_money(value: float | None) -> str: + return f"${value:.2f}" if value is not None else "—" + + +def _print_dry_run( + path: Path, + episode: Episode, + transcript: Transcript, + *, + backend: str, + model: str | None, + max_input_chars: int, + raw: dict | None, +) -> DryRunEstimate: + est = estimate_dry_run( + episode, + transcript, + backend=backend, + model=model, + max_input_chars=max_input_chars, + providers=raw or {}, + ) + console.print( + f"[bold]Dry run[/bold]: {path} — {episode.title if episode else path.stem}" + ) + console.print( + f" input: {est.input_chars:,} chars -> {est.input_tokens:,} tokens" + f" | output est: {est.output_tokens:,} tokens" + f" | total: {est.total_tokens:,} tokens" + ) + if est.chunked: + console.print(f" plan: {est.chunk_count} chunks (over max-input-chars)") + else: + console.print(" plan: single pass") + if backend == "fake": + console.print(" backend fake: no inference call expected - token estimate only") + return est + if est.model_known: + info = get_model(raw or {}, backend, model or "") + console.print(f" model: {model} ({info.name})") + if est.cost_known: + console.print(f" cost: ${est.cost_usd:,.6f}") + else: + console.print(" cost: unknown (no pricing in catalog)") + elif raw is not None: + console.print(f" cost: unknown (model '{model}' not in catalog)") + else: + console.print(" cost: unknown (catalog unavailable)") + return est + + +def _run_nuggets_dry_run( + *, + json_path: Optional[Path], + feed: Optional[str], + all_feeds: bool, + limit: Optional[int], + transcripts_root: Path, + data_dir: Path, + backend: str, + model: str | None, + max_input_chars: int, + quiet: bool, +) -> None: + try: + raw = load_catalog(data_dir) + except CatalogError: + raw = None + if json_path is not None: + path = json_path.expanduser() + if not path.is_file(): + err_console.print(f"[red]File not found:[/red] {path}") + raise typer.Exit(1) + try: + episode, transcript = load_transcript_json(path) + except TranscriptJsonError as exc: + err_console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + _print_dry_run( + path, + episode, + transcript, + backend=backend, + model=model, + max_input_chars=max_input_chars, + raw=raw, + ) + return + + try: + targets = discover_transcript_jsons( + transcripts_root, + feed=None if all_feeds else feed, + ) + except TranscriptJsonError as exc: + err_console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + + if not targets: + err_console.print("[dim]No transcript JSON files found.[/dim]") + raise typer.Exit(1) + + if limit is not None: + targets = targets[:limit] + + total_tokens = 0 + total_cost = 0.0 + costed = 0 + ok = 0 + for target in targets: + try: + episode, transcript = load_transcript_json(target) + except TranscriptJsonError as exc: + err_console.print(f"[red]Skipping[/red] {target}: {exc}") + continue + est = _print_dry_run( + target, + episode, + transcript, + backend=backend, + model=model, + max_input_chars=max_input_chars, + raw=raw, + ) + ok += 1 + total_tokens += est.total_tokens + if est.cost_known: + costed += 1 + total_cost += est.cost_usd + if not quiet: + console.print( + f"[bold]TOTAL[/bold]: {ok} episodes, {total_tokens:,} tokens, " + f"${total_cost:,.6f} ({costed}/{ok} costed)" + ) + if ok != len(targets): + raise typer.Exit(1) + + +def _show_provider_counts(raw: dict) -> None: + supported = sorted(set(available_providers()) & set(catalog_providers(raw))) + if not supported: + console.print("[dim]No configured providers found in the models.dev catalog.[/dim]") + return + table = Table(title="models.dev catalog - configured providers") + table.add_column("Provider") + table.add_column("Name") + table.add_column("Models") + for pid in supported: + entry = raw[pid] + table.add_row( + pid, + entry.get("name", pid), + str(len(list_models(raw, pid))), + ) + console.print(table) + + +def _show_provider_models(raw: dict, *, provider: str, limit: Optional[int]) -> None: + rows = list_models(raw, provider) + if not rows: + err_console.print(f"[red]Provider {provider!r} has no models in the models.dev catalog.[/red]") + raise typer.Exit(1) + if limit is not None: + rows = rows[:limit] + table = Table(title=f"models.dev catalog - {provider}") + table.add_column("Model") + table.add_column("Context") + table.add_column("$/M in") + table.add_column("$/M out") + for model in rows: + table.add_row( + model.name, + _fmt_integer(model.context_length), + _fmt_money(model.cost_input_per_million), + _fmt_money(model.cost_output_per_million), + ) + console.print(table) + + +def _show_models(raw: dict, *, provider: Optional[str], model_id: str) -> None: + matches: list[tuple[str, ModelInfo]] = [] + if provider is not None: + info = get_model(raw, provider, model_id) + if info is not None: + matches.append((provider, info)) + else: + for pid in catalog_providers(raw): + info = get_model(raw, pid, model_id) + if info is not None: + matches.append((pid, info)) + if not matches: + where = f" for provider {provider!r}" if provider is not None else "" + err_console.print(f"[red]Model {model_id!r} is not in the models.dev catalog{where}.[/red]") + raise typer.Exit(1) + table = Table(title=f"models.dev catalog - {model_id}") + table.add_column("Provider") + table.add_column("Model") + table.add_column("Context") + table.add_column("$/M in") + table.add_column("$/M out") + for pid, model in matches: + table.add_row( + pid, + model.name, + _fmt_integer(model.context_length), + _fmt_money(model.cost_input_per_million), + _fmt_money(model.cost_output_per_million), + ) + console.print(table) + + +@app.command("models") +def models_cmd( + provider: Optional[str] = typer.Option( + None, + "--provider", + help="List models for a single provider id (e.g. openrouter, lmstudio)", + ), + model_id: Optional[str] = typer.Option( + None, + "--model", + "-m", + help="Show a specific model id (searches all providers when --provider is omitted)", + ), + limit: Optional[int] = typer.Option(None, "--limit", "-n", help="Cap the number of models listed"), + refresh: bool = typer.Option( + False, "--refresh", help="Re-fetch the models.dev catalog, ignoring the cache" + ), + data_dir: Optional[Path] = typer.Option( + None, "--data-dir", help="Override data directory (models.dev cache location)" + ), +) -> None: + """Inspect the models.dev catalog (metadata only, no inference). + + Shows which registered providers exist in the catalog and how many + models each exposes, lists a provider's models with context window and + USD pricing, or validates a model id (cross-provider search). + """ + settings = load_settings(data_dir=data_dir) + try: + raw = load_catalog(settings.data_dir, refresh=refresh) + except CatalogError as exc: + err_console.print(f"[red]Failed to load models.dev catalog:[/red] {exc}") + raise typer.Exit(1) from exc + + if model_id is not None: + _show_models(raw, provider=provider, model_id=model_id) + return + if provider is not None: + _show_provider_models(raw, provider=provider, limit=limit) + return + _show_provider_counts(raw) + + @auth_app.command("set") def auth_set( backend: str = typer.Argument(..., help="Backend: openrouter, opencode, anthropic, openai"), diff --git a/src/podtx/nuggets.py b/src/podtx/nuggets.py index 5989e95..401a064 100644 --- a/src/podtx/nuggets.py +++ b/src/podtx/nuggets.py @@ -22,10 +22,14 @@ from podtx.format_cmd import TranscriptJsonError, load_transcript_json from podtx.models import Episode, Transcript from podtx.providers import ( + DEFAULT_DRY_OUTPUT_CHARS, Provider, ProviderError, available_providers, build_provider, + estimate_cost, + estimate_tokens, + get_model, normalize_backend, get_spec, ) @@ -342,6 +346,94 @@ def _split_chunks(transcript: Transcript, *, max_input_chars: int) -> list[str]: return pieces +@dataclass(frozen=True) +class DryRunEstimate: + """Token/cost estimate for `podtx nuggets --dry-run` (no inference).""" + + input_chars: int + input_tokens: int + output_tokens: int + total_tokens: int + context_length: int | None + chunked: bool + chunk_count: int + fits: bool | None + cost_usd: float | None + cost_known: bool + model_known: bool + + +def estimate_dry_run( + episode: Episode, + transcript: Transcript, + *, + backend: str, + model: str | None, + max_input_chars: int | None, + providers: dict, + output_chars: int = DEFAULT_DRY_OUTPUT_CHARS, +) -> DryRunEstimate: + """Estimate token usage and cost for a nugget extraction without running it. + + ``providers`` is the raw models.dev provider map (an empty dict disables the + catalog). Estimates are still produced for the offline `fake` backend, but no + pricing applies. ``max_input_chars`` falls back to the nuggets default. + """ + budget = ( + max_input_chars + if max_input_chars is not None + else DEFAULT_NUGGETS_MAX_INPUT_CHARS + ) + input_chars = len(_transcript_text(transcript)) + input_tokens = estimate_tokens(input_chars) + output_tokens = estimate_tokens(output_chars) + total_tokens = input_tokens + output_tokens + chunks = _split_chunks(transcript, max_input_chars=budget) + chunked = len(chunks) > 1 + if backend == "fake": + return DryRunEstimate( + input_chars=input_chars, + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + context_length=None, + chunked=chunked, + chunk_count=len(chunks), + fits=None, + cost_usd=None, + cost_known=False, + model_known=False, + ) + info = get_model(providers, backend, model or "") + model_known = info is not None + fits = None + if info is not None: + if info.context_length is None: + fits = None + else: + fits = input_tokens <= info.context_length + est = estimate_cost( + providers, + backend, + model or "", + input_chars=input_chars, + output_chars=output_chars, + ) + return DryRunEstimate( + input_chars=input_chars, + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + context_length=info.context_length if info is not None else None, + chunked=chunked, + chunk_count=len(chunks), + fits=fits, + cost_usd=est.cost_usd, + cost_known=est.cost_known, + model_known=model_known, + ) + + def _transcript_text(transcript: Transcript) -> str: t = transcript.text.strip() if not t and transcript.segments: diff --git a/src/podtx/providers/__init__.py b/src/podtx/providers/__init__.py index 3e831b8..d386ac1 100644 --- a/src/podtx/providers/__init__.py +++ b/src/podtx/providers/__init__.py @@ -6,6 +6,20 @@ """ from podtx.providers.base import Provider, ProviderError +from podtx.providers.catalog import ( + DEFAULT_DRY_OUTPUT_CHARS, + CatalogError, + CostEstimate, + ModelInfo, + catalog_providers, + estimate_cost, + estimate_tokens, + fetch_catalog, + get_model, + list_models, + load_catalog, + parse_catalog, +) from podtx.providers.registry import ( available_providers, build_provider, @@ -15,11 +29,23 @@ ) __all__ = [ + "DEFAULT_DRY_OUTPUT_CHARS", "Provider", "ProviderError", + "CatalogError", + "CostEstimate", + "ModelInfo", "available_providers", "build_provider", + "catalog_providers", + "estimate_cost", + "estimate_tokens", + "fetch_catalog", + "get_model", "get_spec", + "list_models", + "load_catalog", "normalize_backend", + "parse_catalog", "resolve_api_key", ] \ No newline at end of file diff --git a/src/podtx/providers/catalog.py b/src/podtx/providers/catalog.py new file mode 100644 index 0000000..f0de8d4 --- /dev/null +++ b/src/podtx/providers/catalog.py @@ -0,0 +1,255 @@ +"""models.dev catalog: model metadata (context, pricing) for validation + cost estimates. + +The [models.dev](https://models.dev) catalog is a metadata-only database, **not** an +inference API. ``api.json`` maps provider id -> provider entry with a ``models`` map of +model id -> ``{limit: {context, output}, cost: {input, output}}`` (USD per million tokens). + +The catalog is fetched once and cached to ``/models-cache.json`` so offline use +works from cache; a stale cache is served when the network is unreachable. No silent +network calls happen on the happy path — ``load_catalog`` hits the network only when the +cache is missing, stale, or ``refresh=True``. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import httpx + +from podtx import __version__ + +MODELS_API_URL = "https://models.dev/api.json" +MODELS_CACHE_TTL_SECONDS = 60 * 60 * 24 +DEFAULT_FETCH_TIMEOUT = 30.0 +DEFAULT_DRY_OUTPUT_CHARS = 2000 + + +class CatalogError(Exception): + """Raised when the models.dev catalog cannot be fetched or cached.""" + + +@dataclass(frozen=True) +class ModelInfo: + """Pricing + context facts for one model from the models.dev catalog.""" + + id: str + name: str + context_length: int | None + output_length: int | None + cost_input_per_million: float | None + cost_output_per_million: float | None + + +@dataclass(frozen=True) +class CostEstimate: + """Token + USD estimate for a dry run.""" + + input_tokens: int + output_tokens: int + total_tokens: int + cost_usd: float | None + cost_known: bool + model_known: bool + + +def _get(url: str, timeout: float) -> httpx.Response: + return httpx.get( + url, + timeout=timeout, + headers={"User-Agent": f"podtx/{__version__}"}, + ) + + +def _to_int(value: Any) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _to_float(value: Any) -> float | None: + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _model_info(key: str, value: Any) -> ModelInfo | None: + if not isinstance(value, dict): + return None + limit = value.get("limit") + cost = value.get("cost") + model_id = value.get("id") or key + return ModelInfo( + id=model_id, + name=value.get("name") or model_id, + context_length=_to_int(limit.get("context")) if isinstance(limit, dict) else None, + output_length=_to_int(limit.get("output")) if isinstance(limit, dict) else None, + cost_input_per_million=_to_float(cost.get("input")) if isinstance(cost, dict) else None, + cost_output_per_million=_to_float(cost.get("output")) if isinstance(cost, dict) else None, + ) + + +def parse_catalog(raw: dict) -> dict[str, list[ModelInfo]]: + """Normalize a raw models.dev provider map into provider -> sorted ModelInfo list.""" + parsed: dict[str, list[ModelInfo]] = {} + for provider, entry in raw.items(): + if not isinstance(entry, dict): + continue + models = entry.get("models") + if not isinstance(models, dict): + parsed[provider] = [] + continue + info: list[ModelInfo] = [] + for key, value in models.items(): + model = _model_info(key, value) + if model is not None: + info.append(model) + info.sort(key=lambda m: m.id) + parsed[provider] = info + return parsed + + +def catalog_providers(raw: dict) -> list[str]: + """Sorted provider ids present in a raw models.dev provider map.""" + return sorted(k for k, v in raw.items() if isinstance(v, dict)) + + +def list_models(raw: dict, provider: str) -> list[ModelInfo]: + """Models served by ``provider`` in the catalog, sorted by model id.""" + entry = raw.get(provider) + models = entry.get("models") if isinstance(entry, dict) else None + if not isinstance(models, dict): + return [] + info: list[ModelInfo] = [] + for key, value in models.items(): + model = _model_info(key, value) + if model is not None: + info.append(model) + info.sort(key=lambda m: m.id) + return info + + +def get_model(raw: dict, provider: str, model_id: str) -> ModelInfo | None: + """Look up a model by its catalog key or reported id, or None.""" + entry = raw.get(provider) + models = entry.get("models") if isinstance(entry, dict) else None + if not isinstance(models, dict): + return None + for key, value in models.items(): + if key == model_id: + return _model_info(key, value) + if isinstance(value, dict) and value.get("id") == model_id: + return _model_info(key, value) + return None + + +def estimate_tokens(chars: int) -> int: + """Rough token estimate: ~4 chars per token.""" + return chars // 4 + + +def estimate_cost( + raw: dict, + provider: str, + model_id: str, + *, + input_chars: int, + output_chars: int, +) -> CostEstimate: + """Token + USD estimate for ``model_id`` under ``provider`` in the catalog.""" + input_tokens = estimate_tokens(input_chars) + output_tokens = estimate_tokens(output_chars) + total = input_tokens + output_tokens + model = get_model(raw, provider, model_id) + if model is None: + return CostEstimate(input_tokens, output_tokens, total, None, False, False) + if model.cost_input_per_million is None or model.cost_output_per_million is None: + return CostEstimate(input_tokens, output_tokens, total, None, False, True) + cost = ( + input_tokens / 1e6 * model.cost_input_per_million + + output_tokens / 1e6 * model.cost_output_per_million + ) + return CostEstimate(input_tokens, output_tokens, total, cost, True, True) + + +def fetch_catalog(*, timeout: float = DEFAULT_FETCH_TIMEOUT) -> dict: + """Fetch and parse the raw models.dev provider map; raise CatalogError on failure.""" + try: + resp = _get(MODELS_API_URL, timeout) + except httpx.RequestError as exc: + raise CatalogError(f"models.dev fetch failed: {exc}") from exc + if resp.status_code != 200: + raise CatalogError( + f"models.dev fetch failed (HTTP {resp.status_code}): {resp.text[:200]}" + ) + try: + data = resp.json() + except (json.JSONDecodeError, ValueError) as exc: + raise CatalogError(f"models.dev returned invalid JSON: {resp.text[:200]!r}") from exc + if not isinstance(data, dict): + raise CatalogError("models.dev returned unexpected data shape (expected a dict of providers)") + return data + + +def catalog_cache_path(data_dir: Path) -> Path: + return data_dir / "models-cache.json" + + +def _read_cache(data_dir: Path) -> dict | None: + try: + raw = json.loads(catalog_cache_path(data_dir).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(raw, dict) or not isinstance(raw.get("data"), dict): + return None + return raw + + +def _cache_fresh(cached: dict, ttl_seconds: float) -> bool: + fetched = cached.get("fetched_at") + if not isinstance(fetched, str): + return False + try: + when = datetime.fromisoformat(fetched) + except ValueError: + return False + return datetime.now(timezone.utc) - when <= timedelta(seconds=ttl_seconds) + + +def _write_cache(data_dir: Path, data: dict) -> None: + payload = { + "fetched_at": datetime.now(timezone.utc).isoformat(), + "data": data, + } + path = catalog_cache_path(data_dir) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +def load_catalog( + data_dir: Path, + *, + refresh: bool = False, + timeout: float = DEFAULT_FETCH_TIMEOUT, + ttl_seconds: float = MODELS_CACHE_TTL_SECONDS, +) -> dict: + """Raw models.dev provider map, served from fresh cache, network, or stale cache. + + Ordering: fresh cache -> fetch (persisted) -> stale cache -> CatalogError. + """ + cached = _read_cache(data_dir) + if cached is not None and not refresh and _cache_fresh(cached, ttl_seconds): + return cached["data"] + try: + data = fetch_catalog(timeout=timeout) + except CatalogError: + if cached is not None: + return cached["data"] + raise + _write_cache(data_dir, data) + return data \ No newline at end of file diff --git a/tests/test_catalog.py b/tests/test_catalog.py new file mode 100644 index 0000000..5439ce1 --- /dev/null +++ b/tests/test_catalog.py @@ -0,0 +1,367 @@ +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import httpx +import pytest + +from podtx.providers import catalog as catalog_mod +from podtx.providers.catalog import ( + CatalogError, + CostEstimate, + ModelInfo, + catalog_providers, + estimate_cost, + estimate_tokens, + fetch_catalog, + get_model, + list_models, + load_catalog, + parse_catalog, +) + +FAKE_API = { + "lmstudio": { + "id": "lmstudio", + "name": "LM Studio", + "models": { + "gpt-oss-20b": { + "id": "openai/gpt-oss-20b", + "name": "GPT-OSS 20B", + "limit": {"context": 131072, "output": 32768}, + "cost": {"input": 0.0, "output": 0.0}, + }, + "qwen/qwen2.5-14b": { + "id": "qwen/qwen2.5-14b", + "name": "Qwen 2.5 14B", + "limit": {"context": 32768}, + }, + "not-a-dict": "skip-me", + }, + }, + "openrouter": { + "id": "openrouter", + "name": "OpenRouter", + "models": { + "anthropic/claude-sonnet-4": { + "id": "anthropic/claude-sonnet-4", + "name": "Claude Sonnet 4", + "limit": {"context": 200000, "output": 64000}, + "cost": {"input": 3.0, "output": 15.0}, + }, + "openai/gpt-4o-mini": { + "id": "openai/gpt-4o-mini", + "name": "GPT-4o mini", + "limit": {"context": 128000}, + "cost": {"input": 0.15, "output": 0.6}, + }, + "mistral/unknown-cost": { + "id": "mistral/unknown-cost", + "name": "Unknown Cost", + "limit": {"context": 32000}, + "cost": {"input": 0.2}, + }, + }, + }, + "openai": {"id": "openai", "name": "OpenAI", "models": {}}, + "weird": "not-a-provider", +} + + +def _with_fetch(monkeypatch, payload, raises=None): + def fake_get(url, timeout): + assert url == catalog_mod.MODELS_API_URL + if raises is not None: + raise raises + return payload + + monkeypatch.setattr(catalog_mod, "_get", fake_get) + + +# --- parse / lookup --------------------------------------------------------- + + +def test_parse_catalog_shape() -> None: + parsed = parse_catalog(FAKE_API) + assert set(parsed) == {"lmstudio", "openrouter", "openai"} + lm = parsed["lmstudio"] + assert len(lm) == 2 + assert lm[0].id == "openai/gpt-oss-20b" + assert lm[0].name == "GPT-OSS 20B" + assert lm[0].context_length == 131072 + assert lm[0].output_length == 32768 + assert lm[0].cost_input_per_million == 0.0 + assert lm[0].cost_output_per_million == 0.0 + qwen = lm[1] + assert qwen.id == "qwen/qwen2.5-14b" + assert qwen.output_length is None + assert qwen.cost_input_per_million is None + assert parsed["openai"] == [] + + +def test_catalog_providers_sorted() -> None: + assert catalog_providers(FAKE_API) == ["lmstudio", "openai", "openrouter"] + + +def test_parse_catalog_non_dict_provider_skipped() -> None: + assert "weird" not in parse_catalog(FAKE_API) + + +def test_list_models_sorted_by_id() -> None: + ids = [m.id for m in list_models(FAKE_API, "lmstudio")] + assert ids == ["openai/gpt-oss-20b", "qwen/qwen2.5-14b"] + + +def test_list_models_unknown_provider_empty() -> None: + assert list_models(FAKE_API, "nope") == [] + + +def test_list_models_non_dict_provider_empty() -> None: + assert list_models(FAKE_API, "weird") == [] + assert list_models(FAKE_API, "openai") == [] + + +def test_get_model_by_key() -> None: + m = get_model(FAKE_API, "lmstudio", "gpt-oss-20b") + assert m is not None + assert m.id == "openai/gpt-oss-20b" + + +def test_get_model_by_reported_id() -> None: + m = get_model(FAKE_API, "lmstudio", "openai/gpt-oss-20b") + assert m is not None + assert m.context_length == 131072 + + +def test_get_model_missing() -> None: + assert get_model(FAKE_API, "lmstudio", "nope") is None + assert get_model(FAKE_API, "nope", "gpt-oss-20b") is None + assert get_model(FAKE_API, "weird", "gpt-oss-20b") is None + assert get_model(FAKE_API, "openai", "anything") is None + assert get_model(FAKE_API, "lmstudio", "not-a-dict") is None + + +# --- token / cost estimates ------------------------------------------------- + + +def test_estimate_tokens() -> None: + assert estimate_tokens(4096) == 1024 + assert estimate_tokens(1) == 0 + assert estimate_tokens(0) == 0 + + +def test_estimate_cost_known_pricing() -> None: + est = estimate_cost( + FAKE_API, + "openrouter", + "anthropic/claude-sonnet-4", + input_chars=4000, + output_chars=2000, + ) + assert est.input_tokens == 1000 + assert est.output_tokens == 500 + assert est.total_tokens == 1500 + assert est.model_known is True + assert est.cost_known is True + assert est.cost_usd == pytest.approx(0.0105) + + +def test_estimate_cost_zero_pricing() -> None: + est = estimate_cost( + FAKE_API, "lmstudio", "gpt-oss-20b", input_chars=1000, output_chars=1000 + ) + assert est.cost_known is True + assert est.cost_usd == 0.0 + + +def test_estimate_cost_unknown_model() -> None: + est = estimate_cost( + FAKE_API, "openrouter", "nope/x", input_chars=1000, output_chars=500 + ) + assert est.input_tokens == 250 + assert est.output_tokens == 125 + assert est.total_tokens == 375 + assert est.model_known is False + assert est.cost_usd is None + + +def test_estimate_cost_partial_pricing_treated_unknown() -> None: + est = estimate_cost( + FAKE_API, "openrouter", "mistral/unknown-cost", input_chars=1000, output_chars=500 + ) + assert est.model_known is True + assert est.cost_known is False + assert est.cost_usd is None + + +def test_estimate_cost_no_pricing() -> None: + est = estimate_cost( + FAKE_API, "lmstudio", "qwen/qwen2.5-14b", input_chars=1000, output_chars=500 + ) + assert est.model_known is True + assert est.cost_known is False + + +# --- fetch ------------------------------------------------------------------ + + +def test_fetch_catalog_ok(monkeypatch) -> None: + _with_fetch(monkeypatch, httpx.Response(200, json=FAKE_API)) + assert fetch_catalog() == FAKE_API + + +def test_fetch_catalog_http_error(monkeypatch) -> None: + _with_fetch(monkeypatch, httpx.Response(500, text="boom")) + with pytest.raises(CatalogError): + fetch_catalog() + + +def test_fetch_catalog_bad_json(monkeypatch) -> None: + _with_fetch(monkeypatch, httpx.Response(200, text="not-json{{")) + with pytest.raises(CatalogError): + fetch_catalog() + + +def test_fetch_catalog_not_dict(monkeypatch) -> None: + _with_fetch(monkeypatch, httpx.Response(200, json=["not", "a", "dict"])) + with pytest.raises(CatalogError): + fetch_catalog() + + +def test_fetch_catalog_request_error(monkeypatch) -> None: + _with_fetch(monkeypatch, None, raises=httpx.ConnectError("boom")) + with pytest.raises(CatalogError): + fetch_catalog() + + +# --- cache / load ----------------------------------------------------------- + + +def _cache_path(tmp_path: Path) -> Path: + return catalog_mod.catalog_cache_path(tmp_path) + + +def _write_cache(tmp_path: Path, data, age_hours=0.0) -> None: + fetched = datetime.now(timezone.utc) - timedelta(hours=age_hours) + _cache_path(tmp_path).write_text( + json.dumps({"fetched_at": fetched.isoformat(), "data": data}), encoding="utf-8" + ) + + +def test_load_no_cache_fetches_and_persists(tmp_path, monkeypatch) -> None: + _with_fetch(monkeypatch, httpx.Response(200, json=FAKE_API)) + raw = load_catalog(tmp_path, ttl_seconds=86400) + assert raw == FAKE_API + cached = json.loads(_cache_path(tmp_path).read_text(encoding="utf-8")) + assert cached["data"] == FAKE_API + assert "fetched_at" in cached + + +def test_load_fresh_cache_does_not_fetch(tmp_path, monkeypatch) -> None: + _write_cache(tmp_path, FAKE_API, age_hours=1) + + def boom(url, timeout): + raise AssertionError("should not fetch") + + monkeypatch.setattr(catalog_mod, "_get", boom) + raw = load_catalog(tmp_path, ttl_seconds=86400) + assert raw == FAKE_API + + +def test_load_stale_cache_serves_stale_when_unreachable(tmp_path, monkeypatch) -> None: + _write_cache(tmp_path, FAKE_API, age_hours=25) + _with_fetch(monkeypatch, None, raises=httpx.ConnectError("offline")) + raw = load_catalog(tmp_path, ttl_seconds=86400) + assert raw == FAKE_API + + +def test_load_no_cache_unreachable_raises(tmp_path, monkeypatch) -> None: + _with_fetch(monkeypatch, None, raises=httpx.ConnectError("offline")) + with pytest.raises(CatalogError): + load_catalog(tmp_path, ttl_seconds=86400) + + +def test_load_stale_cache_refreshes_and_persists(tmp_path, monkeypatch) -> None: + newer = {"openrouter": {"id": "openrouter", "name": "OpenRouter", "models": {}}} + _write_cache(tmp_path, FAKE_API, age_hours=25) + _with_fetch(monkeypatch, httpx.Response(200, json=newer)) + raw = load_catalog(tmp_path, ttl_seconds=86400) + assert raw == newer + cached = json.loads(_cache_path(tmp_path).read_text(encoding="utf-8")) + assert cached["data"] == newer + + +def test_load_refresh_forces_fetch(tmp_path, monkeypatch) -> None: + newer = {"openai": {"id": "openai", "name": "OpenAI", "models": {}}} + _write_cache(tmp_path, FAKE_API, age_hours=0) + _with_fetch(monkeypatch, httpx.Response(200, json=newer)) + raw = load_catalog(tmp_path, ttl_seconds=86400, refresh=True) + assert raw == newer + + +def test_load_malformed_cache_fetches(tmp_path, monkeypatch) -> None: + _cache_path(tmp_path).write_text("not-json{{", encoding="utf-8") + _with_fetch(monkeypatch, httpx.Response(200, json=FAKE_API)) + assert load_catalog(tmp_path, ttl_seconds=86400) == FAKE_API + + +def test_load_cache_missing_keys_fetches(tmp_path, monkeypatch) -> None: + _cache_path(tmp_path).write_text(json.dumps({"foo": 1}), encoding="utf-8") + _with_fetch(monkeypatch, httpx.Response(200, json=FAKE_API)) + assert load_catalog(tmp_path, ttl_seconds=86400) == FAKE_API + + +def test_load_cache_bad_fetched_at_fetches(tmp_path, monkeypatch) -> None: + _cache_path(tmp_path).write_text( + json.dumps({"fetched_at": "garbage", "data": FAKE_API}), encoding="utf-8" + ) + _with_fetch(monkeypatch, httpx.Response(200, json=FAKE_API)) + assert load_catalog(tmp_path, ttl_seconds=86400) == FAKE_API + + +def test_load_cache_data_not_dict_fetches(tmp_path, monkeypatch) -> None: + _cache_path(tmp_path).write_text( + json.dumps({"fetched_at": "garbage", "data": [1, 2]}), encoding="utf-8" + ) + _with_fetch(monkeypatch, httpx.Response(200, json=FAKE_API)) + assert load_catalog(tmp_path, ttl_seconds=86400) == FAKE_API + + +def test_load_stale_threshold(tmp_path, monkeypatch) -> None: + _write_cache(tmp_path, FAKE_API, age_hours=59 / 60) + _with_fetch(monkeypatch, None, raises=httpx.ConnectError("should not fetch")) + assert load_catalog(tmp_path, ttl_seconds=3600) == FAKE_API + _write_cache(tmp_path, FAKE_API, age_hours=61 / 60) + _with_fetch(monkeypatch, httpx.Response(200, json=FAKE_API)) + assert load_catalog(tmp_path, ttl_seconds=3600) == FAKE_API + + +def test_model_info_is_frozen() -> None: + m = ModelInfo("a", "b", None, None, None, None) + with pytest.raises(Exception): + m.id = "x" # type: ignore[misc] + + +def test_cost_estimate_is_frozen() -> None: + c = CostEstimate(1, 2, 3, None, False, False) + with pytest.raises(Exception): + c.total_tokens = 99 # type: ignore[misc] + +def test_load_cache_fetched_at_not_string_fetches(tmp_path, monkeypatch) -> None: + _cache_path(tmp_path).write_text( + json.dumps({"fetched_at": 12345, "data": FAKE_API}), encoding="utf-8" + ) + _with_fetch(monkeypatch, httpx.Response(200, json=FAKE_API)) + assert load_catalog(tmp_path, ttl_seconds=86400) == FAKE_API + + +def test_parse_catalog_non_dict_models_empty() -> None: + parsed = parse_catalog({"broken": {"id": "broken", "name": "Broken", "models": "oops"}}) + assert parsed["broken"] == [] + + +def test_get_real_http_error(monkeypatch) -> None: + with pytest.raises(httpx.RequestError): + catalog_mod._get("http://127.0.0.1:1/", timeout=0.5) diff --git a/tests/test_cli_models.py b/tests/test_cli_models.py new file mode 100644 index 0000000..4af764a --- /dev/null +++ b/tests/test_cli_models.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from podtx.cli import app +from podtx.providers.catalog import CatalogError + +runner = CliRunner() + +FAKE_API = { + "lmstudio": { + "id": "lmstudio", + "name": "LM Studio", + "models": { + "gpt-oss-20b": { + "id": "openai/gpt-oss-20b", + "name": "GPT-OSS 20B", + "limit": {"context": 131072, "output": 32768}, + "cost": {"input": 0.0, "output": 0.0}, + }, + "qwen/qwen2.5-14b": { + "id": "qwen/qwen2.5-14b", + "name": "Qwen 2.5 14B", + "limit": {"context": 32768}, + }, + "vague/none": { + "id": "vague/none", + "name": "Vague None", + }, + }, + }, + "openrouter": { + "id": "openrouter", + "name": "OpenRouter", + "models": { + "anthropic/claude-sonnet-4": { + "id": "anthropic/claude-sonnet-4", + "name": "Claude Sonnet 4", + "limit": {"context": 200000, "output": 64000}, + "cost": {"input": 3.0, "output": 15.0}, + }, + "openai/gpt-4o-mini": { + "id": "openai/gpt-4o-mini", + "name": "GPT-4o mini", + "limit": {"context": 128000}, + "cost": {"input": 0.15, "output": 0.6}, + }, + }, + }, + "openai": {"id": "openai", "name": "OpenAI", "models": {}}, +} + + +def _load_fake(tmp_path: Path): + return lambda data_dir, *, refresh=False, timeout=120.0, ttl_seconds=86400: FAKE_API + + +def test_models_lists_supported_counts(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke(app, ["models", "--data-dir", str(tmp_path)]) + assert result.exit_code == 0, result.stdout + result.stderr + assert "lmstudio" in result.stdout and "2" in result.stdout + assert "openrouter" in result.stdout and "OpenRouter" in result.stdout + assert "anthropic" not in result.stdout + + +def test_models_provider_listing(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke(app, ["models", "--provider", "openrouter", "--data-dir", str(tmp_path)]) + assert result.exit_code == 0, result.stdout + result.stderr + assert "Claude Sonnet 4" in result.stdout + assert "200,000" in result.stdout or "200000" in result.stdout + assert "GPT-4o mini" in result.stdout + assert "Qwen 2.5 14B" not in result.stdout + + +def test_models_provider_limit(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, ["models", "--provider", "openrouter", "--limit", "1", "--data-dir", str(tmp_path)] + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "Claude Sonnet 4" in result.stdout + assert "GPT-4o mini" not in result.stdout + + +def test_models_unknown_provider(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke(app, ["models", "--provider", "nope", "--data-dir", str(tmp_path)]) + assert result.exit_code != 0 + assert "nope" in result.stdout or "nope" in result.stderr + + +def test_models_model_found_in_provider(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, + ["models", "--provider", "lmstudio", "--model", "qwen/qwen2.5-14b", "--data-dir", str(tmp_path)], + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "Qwen 2.5 14B" in result.stdout + assert "32,768" in result.stdout or "32768" in result.stdout + + +def test_models_model_search_across_providers(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke(app, ["models", "--model", "anthropic/claude-sonnet-4", "--data-dir", str(tmp_path)]) + assert result.exit_code == 0, result.stdout + result.stderr + assert "openrouter" in result.stdout + assert "Claude Sonnet 4" in result.stdout + + +def test_models_model_not_found(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke(app, ["models", "--model", "nope/nope", "--data-dir", str(tmp_path)]) + assert result.exit_code != 0 + assert "nope/nope" in result.stdout or "nope/nope" in result.stderr + + +def test_models_model_not_found_in_provider(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, + ["models", "--provider", "openai", "--model", "gpt-oss-20b", "--data-dir", str(tmp_path)], + ) + assert result.exit_code != 0 + + +def test_models_catalog_unavailable(tmp_path, monkeypatch) -> None: + def boom(data_dir, *, refresh=False, timeout=120.0, ttl_seconds=86400): + raise CatalogError("no cache and offline") + + monkeypatch.setattr("podtx.cli.load_catalog", boom) + result = runner.invoke(app, ["models", "--data-dir", str(tmp_path)]) + assert result.exit_code != 0 + assert "no cache and offline" in result.stdout or "no cache and offline" in result.stderr + + +def test_models_refresh_forces_fetch(tmp_path, monkeypatch) -> None: + calls = {} + + def record(data_dir, *, refresh=False, timeout=120.0, ttl_seconds=86400): + calls["refresh"] = refresh + return FAKE_API + + monkeypatch.setattr("podtx.cli.load_catalog", record) + result = runner.invoke(app, ["models", "--refresh", "--data-dir", str(tmp_path)]) + assert result.exit_code == 0, result.stdout + result.stderr + assert calls["refresh"] is True + +def test_models_provider_listing_shows_unknowns(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke(app, ["models", "--provider", "lmstudio", "--data-dir", str(tmp_path)]) + assert result.exit_code == 0, result.stdout + result.stderr + assert "Vague None" in result.stdout + assert "—" in result.stdout + + +def test_models_no_configured_providers(tmp_path, monkeypatch) -> None: + monkeypatch.setattr( + "podtx.cli.load_catalog", + lambda data_dir, *, refresh=False, timeout=120.0, ttl_seconds=86400: { + "some-other-vendor": {"id": "some-other-vendor", "name": "Other", "models": {}} + }, + ) + result = runner.invoke(app, ["models", "--data-dir", str(tmp_path)]) + assert result.exit_code == 0, result.stdout + result.stderr + assert "No configured providers found" in result.stdout diff --git a/tests/test_cli_nuggets_dryrun.py b/tests/test_cli_nuggets_dryrun.py new file mode 100644 index 0000000..1a9ee42 --- /dev/null +++ b/tests/test_cli_nuggets_dryrun.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path + +from typer.testing import CliRunner + +from podtx.cli import app +from podtx.models import Episode, Segment, Transcript +from podtx.providers.catalog import CatalogError +from podtx.writers import write_outputs + +runner = CliRunner() + +FAKE_API = { + "openrouter": { + "id": "openrouter", + "name": "OpenRouter", + "models": { + "anthropic/claude-sonnet-4": { + "id": "anthropic/claude-sonnet-4", + "name": "Claude Sonnet 4", + "limit": {"context": 200000, "output": 64000}, + "cost": {"input": 3.0, "output": 15.0}, + }, + "openai/gpt-4o-mini": { + "id": "openai/gpt-4o-mini", + "name": "GPT-4o mini", + "limit": {"context": 128000}, + "cost": {"input": 0.15, "output": 0.6}, + }, + }, + }, + "lmstudio": { + "id": "lmstudio", + "name": "LM Studio", + "models": { + "openai/gpt-oss-20b": { + "id": "openai/gpt-oss-20b", + "name": "GPT-OSS 20B", + "limit": {"context": 131072, "output": 32768}, + "cost": {"input": 0.0, "output": 0.0}, + }, + "qwen/qwen2.5-14b": { + "id": "qwen/qwen2.5-14b", + "name": "Qwen 2.5 14B", + "limit": {"context": 32768}, + }, + }, + }, +} + + +def _episode() -> Episode: + return Episode( + guid="fake-guid-1", + title="Fake Episode Title", + enclosure_url="https://example.com/ep.mp3", + published_at=datetime(2026, 3, 15, tzinfo=timezone.utc), + episode_num=42, + show_title="Fake Show", + link="https://example.com/ep", + ) + + +def _transcript() -> Transcript: + return Transcript( + text="First sentence is overview. Second sentence also overview. Third is a key point. Fourth more. Fifth extra.", + segments=[ + Segment(0.0, 1.5, "First sentence is overview."), + Segment(2.0, 3.5, "Second sentence also overview."), + Segment(10.0, 12.0, "Third is a key point."), + Segment(65.0, 70.0, "Fourth more."), + Segment(120.0, 125.0, "Fifth extra."), + ], + language="en", + model="fake-model", + engine="fake", + ) + + +def _write_transcript(tmp_path: Path, basename: str = "ep") -> Path: + write_outputs( + out_dir=tmp_path, + basename=basename, + episode=_episode(), + transcript=_transcript(), + formats=("txt", "json"), + readable=False, + cleanup=False, + ) + return tmp_path / f"{basename}.json" + + +def _load_fake(_tmp_path: Path): + def fake_load(data_dir, *, refresh=False, timeout=120.0, ttl_seconds=86400): + return FAKE_API + + return fake_load + + +def test_dry_run_single_no_sidecar_no_call(tmp_path, monkeypatch) -> None: + path = _write_transcript(tmp_path) + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, + [ + "nuggets", + str(path), + "--backend", + "openrouter", + "--model", + "anthropic/claude-sonnet-4", + "--dry-run", + ], + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "dry run" in result.stdout.lower() + assert "Claude Sonnet 4" in result.stdout + assert not (tmp_path / "ep.nuggets.json").exists() + assert not (tmp_path / "ep.nuggets.md").exists() + + +def test_dry_run_single_cost_estimate(tmp_path, monkeypatch) -> None: + path = _write_transcript(tmp_path) + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, ["nuggets", str(path), "--backend", "openrouter", "--model", "anthropic/claude-sonnet-4", "--dry-run"] + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "tokens" in result.stdout + assert "cost" in result.stdout + + +def test_dry_run_fake_backend(tmp_path, monkeypatch) -> None: + path = _write_transcript(tmp_path) + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke(app, ["nuggets", str(path), "--backend", "fake", "--dry-run"]) + assert result.exit_code == 0, result.stdout + result.stderr + assert "no inference" in result.stdout.lower() + + +def test_dry_run_catalog_unavailable(tmp_path, monkeypatch) -> None: + path = _write_transcript(tmp_path) + + def boom(data_dir, *, refresh=False, timeout=120.0, ttl_seconds=86400): + raise CatalogError("no cache and offline") + + monkeypatch.setattr("podtx.cli.load_catalog", boom) + result = runner.invoke( + app, ["nuggets", str(path), "--backend", "openrouter", "--model", "anthropic/claude-sonnet-4", "--dry-run"] + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "catalog unavailable" in result.stdout.lower() + assert "cost: unknown" in result.stdout.lower() or "cost: unknown" in result.stderr.lower() + + +def test_dry_run_unknown_model(tmp_path, monkeypatch) -> None: + path = _write_transcript(tmp_path) + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, ["nuggets", str(path), "--backend", "openrouter", "--model", "nope/x", "--dry-run"] + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "not in catalog" in result.stdout.lower() + + +def test_dry_run_single_bad_json(tmp_path, monkeypatch) -> None: + bad = tmp_path / "bad.json" + bad.write_text("{{{{", encoding="utf-8") + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke(app, ["nuggets", str(bad), "--backend", "fake", "--dry-run"]) + assert result.exit_code != 0 + assert "Could not read transcript JSON" in result.stdout + result.stderr + + +def test_dry_run_feed_sums(tmp_path, monkeypatch) -> None: + root = tmp_path / "transcripts" / "myshow" + root.mkdir(parents=True, exist_ok=True) + _write_transcript(root, basename="ep0") + _write_transcript(root, basename="ep1") + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, + [ + "nuggets", + "--feed", + "myshow", + "--data-dir", + str(tmp_path), + "--backend", + "openrouter", + "--model", + "anthropic/claude-sonnet-4", + "--dry-run", + ], + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "TOTAL" in result.stdout + assert "2 episodes" in result.stdout + + +def test_dry_run_feed_no_transcripts(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, ["nuggets", "--feed", "nope", "--data-dir", str(tmp_path), "--dry-run"] + ) + assert result.exit_code != 0 + +def test_dry_run_no_pricing_in_catalog(tmp_path, monkeypatch) -> None: + path = _write_transcript(tmp_path) + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, ["nuggets", str(path), "--backend", "lmstudio", "--model", "qwen/qwen2.5-14b", "--dry-run"] + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "no pricing in catalog" in result.stdout + + +def test_dry_run_single_chunked(tmp_path, monkeypatch) -> None: + path = _write_transcript(tmp_path) + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, + [ + "nuggets", + str(path), + "--backend", + "openrouter", + "--model", + "anthropic/claude-sonnet-4", + "--max-input-chars", + "5", + "--dry-run", + ], + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "chunks" in result.stdout + + +def test_dry_run_feed_skips_broken(tmp_path, monkeypatch) -> None: + root = tmp_path / "transcripts" / "myshow" + root.mkdir(parents=True, exist_ok=True) + _write_transcript(root, basename="ep0") + _write_transcript(root, basename="ep1") + (root / "ep2.json").write_text("{{{{", encoding="utf-8") + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, ["nuggets", "--feed", "myshow", "--data-dir", str(tmp_path), "--dry-run"] + ) + assert result.exit_code != 0 + assert "Skipping" in result.stdout + result.stderr + + +def test_dry_run_feed_quiet(tmp_path, monkeypatch) -> None: + root = tmp_path / "transcripts" / "myshow" + root.mkdir(parents=True, exist_ok=True) + _write_transcript(root, basename="ep0") + _write_transcript(root, basename="ep1") + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, + ["nuggets", "--feed", "myshow", "--data-dir", str(tmp_path), "--backend", "fake", "--dry-run", "--quiet"], + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "TOTAL" not in result.stdout + + +def test_dry_run_single_file_not_found(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + missing = tmp_path / "missing.json" + result = runner.invoke(app, ["nuggets", str(missing), "--dry-run"]) + assert result.exit_code != 0 + assert "File not found" in result.stdout + result.stderr + + +def test_dry_run_feed_empty_transcripts(tmp_path, monkeypatch) -> None: + (tmp_path / "transcripts" / "myshow").mkdir(parents=True, exist_ok=True) + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke(app, ["nuggets", "--feed", "myshow", "--data-dir", str(tmp_path), "--dry-run"]) + assert result.exit_code != 0 + assert "No transcript JSON files found" in result.stdout + result.stderr + + +def test_dry_run_feed_limit(tmp_path, monkeypatch) -> None: + root = tmp_path / "transcripts" / "myshow" + root.mkdir(parents=True, exist_ok=True) + _write_transcript(root, basename="ep0") + _write_transcript(root, basename="ep1") + monkeypatch.setattr("podtx.cli.load_catalog", _load_fake(tmp_path)) + result = runner.invoke( + app, ["nuggets", "--feed", "myshow", "--data-dir", str(tmp_path), "--limit", "1", "--dry-run"] + ) + assert result.exit_code == 0, result.stdout + result.stderr + assert "TOTAL" in result.stdout + assert "1 episodes" in result.stdout diff --git a/tests/test_nuggets.py b/tests/test_nuggets.py index eed7025..8b0da71 100644 --- a/tests/test_nuggets.py +++ b/tests/test_nuggets.py @@ -34,9 +34,10 @@ _verify_quotes, _write_nugget_files, extract_nuggets_transcript, + estimate_dry_run, nuggets_many, ) -from podtx.providers import ProviderError +from podtx.providers import ProviderError, estimate_tokens from podtx.writers import write_outputs @@ -794,4 +795,180 @@ def test_nuggets_many_counts_provider_errors(tmp_path: Path) -> None: def test_batch_result_defaults() -> None: r = BatchNuggetsResult() assert r.ok == 0 and r.failed == 0 and r.skipped == 0 - assert r.written == [] and r.errors == [] \ No newline at end of file + assert r.written == [] and r.errors == [] + +def _dry_api() -> dict: + return { + "openrouter": { + "id": "openrouter", + "name": "OpenRouter", + "models": { + "anthropic/claude-sonnet-4": { + "id": "anthropic/claude-sonnet-4", + "name": "Claude Sonnet 4", + "limit": {"context": 200000, "output": 64000}, + "cost": {"input": 3.0, "output": 15.0}, + }, + "tiny/ctx": { + "id": "tiny/ctx", + "name": "Tiny Context", + "limit": {"context": 10}, + }, + "openrouter/contextless": { + "id": "openrouter/contextless", + "name": "No Ctx", + "limit": {"output": 1000}, + }, + }, + } + } + + +def test_estimate_dry_run_fake() -> None: + est = estimate_dry_run( + _episode(), + _transcript(), + backend="fake", + model=None, + max_input_chars=100_000, + providers=_dry_api(), + ) + assert est.input_chars == len(_transcript().text.rstrip()) + assert est.input_tokens == estimate_tokens(est.input_chars) + assert est.output_tokens == estimate_tokens(2000) + assert est.total_tokens == est.input_tokens + est.output_tokens + assert est.chunked is False + assert est.chunk_count == 1 + assert est.fits is None + assert est.cost_usd is None + assert est.cost_known is False + assert est.model_known is False + + +def test_estimate_dry_run_known_model_fits() -> None: + est = estimate_dry_run( + _episode(), + _transcript(), + backend="openrouter", + model="anthropic/claude-sonnet-4", + max_input_chars=100_000, + providers=_dry_api(), + ) + assert est.fits is True + assert est.chunked is False + assert est.model_known is True + assert est.cost_known is True + assert est.cost_usd == pytest.approx( + est.input_tokens / 1e6 * 3.0 + est.output_tokens / 1e6 * 15.0 + ) + + +def test_estimate_dry_run_known_model_too_small() -> None: + est = estimate_dry_run( + _episode(), + _transcript(), + backend="openrouter", + model="tiny/ctx", + max_input_chars=100_000, + providers=_dry_api(), + ) + assert est.fits is False + assert est.chunked is False + assert est.model_known is True + assert est.cost_known is False + + +def test_estimate_dry_run_unknown_model() -> None: + est = estimate_dry_run( + _episode(), + _transcript(), + backend="openrouter", + model="nope/x", + max_input_chars=100_000, + providers=_dry_api(), + ) + assert est.fits is None + assert est.model_known is False + assert est.cost_known is False + assert est.cost_usd is None + + +def test_estimate_dry_run_chunked() -> None: + tx = Transcript( + text=" ".join(["word"] * 60_000), + segments=[Segment(float(i), float(i) + 1, "word") for i in range(60_000)], + language="en", + model="m", + engine="fake", + ) + est = estimate_dry_run( + _episode(), + tx, + backend="openrouter", + model="anthropic/claude-sonnet-4", + max_input_chars=10_000, + providers=_dry_api(), + ) + assert est.chunked is True + assert est.chunk_count > 1 + assert est.fits is True + + +def test_estimate_dry_run_no_providers() -> None: + est = estimate_dry_run( + _episode(), + _transcript(), + backend="openrouter", + model="anthropic/claude-sonnet-4", + max_input_chars=100_000, + providers={}, + ) + assert est.model_known is False + assert est.fits is None + assert est.cost_usd is None + assert est.input_tokens == estimate_tokens(est.input_chars) + + +def test_estimate_dry_run_default_max_input_chars() -> None: + tx = Transcript( + text=" ".join(["word"] * 60_000), + segments=[Segment(float(i), float(i) + 1, "word") for i in range(60_000)], + language="en", + model="m", + engine="fake", + ) + est = estimate_dry_run( + _episode(), + tx, + backend="fake", + model=None, + max_input_chars=None, + providers=_dry_api(), + ) + assert est.chunked is True + assert est.input_tokens == estimate_tokens(est.input_chars) + + +def test_split_text_no_pieces_when_empty() -> None: + assert _split_text(" ", max_chars=10, overlap_chars=2) == [] + assert _split_text("", max_chars=10, overlap_chars=2) == [] + + +def test_split_text_exhausts_overlap_without_break() -> None: + word = "b" * 48 + assert _split_text("aa " + word, max_chars=50, overlap_chars=6) == ["aa", "aa " + word] + + +def test_estimate_dry_run_contextless_model() -> None: + est = estimate_dry_run( + _episode(), + _transcript(), + backend="openrouter", + model="openrouter/contextless", + max_input_chars=100_000, + providers=_dry_api(), + ) + assert est.model_known is True + assert est.context_length is None + assert est.fits is None + assert est.cost_usd is None