diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md index 802acf9..aebb516 100644 --- a/GETTING_STARTED.md +++ b/GETTING_STARTED.md @@ -1,5 +1,24 @@ # Getting Started with InsightWeaver +> **Superseded 2026-08-31 (backlog task 012).** The briefing product this +> document describes -- `brief`, beats, questions, predictions, frames, +> synthesis and rendering -- has been **deleted**. Roughly 8,900 lines of source +> and most of the test suite went with it, in a single commit, because the +> operator's call was that git history is the rollback path and dead code in the +> tree reads as current. `insightweaver` now exposes one command, `sources`. +> +> What survives is ingestion: `src/sources/`, `src/rss/`, `src/processors/`, the +> feed config, and five modules ported for tiers not yet built +> (`src/matching/entity_matcher.py`, `src/matching/coverage_probe.py`, +> `src/llm/claude_client.py`, `src/utils/cadence.py`, +> `src/processors/deduplicator.py`). +> +> **Everything below this banner describes the deleted product and has not been +> rewritten.** Rewriting it needs the new pipeline to exist first, which task 012 +> explicitly puts out of scope. Read it as history until it is replaced. + +--- + This guide walks you through downloading, installing, and running InsightWeaver on your computer. --- diff --git a/Makefile b/Makefile index b446b2b..e1e8db1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install install-dev test lint format fmt typecheck check pre-commit clean clean-all coverage run-brief run-forecast db-migrate db-migrate-down db-reset update-deps +.PHONY: help install install-dev test lint format fmt typecheck check pre-commit clean clean-all coverage db-drop-briefing update-deps # Tool resolution (added 2026-08-24). # Before this, every recipe called bare `pytest` / `ruff` / `mypy`, which only @@ -43,14 +43,9 @@ help: @echo " make pre-commit Run all pre-commit hooks" @echo " make check Run all checks (lint + typecheck + test)" @echo "" - @echo "Application:" - @echo " make run-brief Run intelligence brief" - @echo " make run-forecast Run forecast generation" - @echo "" @echo "Database:" - @echo " make db-migrate Run database migrations" - @echo " make db-migrate-down Rollback database migrations" - @echo " make db-reset Reset database (WARNING: deletes all data)" + @echo " make db-drop-briefing Drop the deleted briefing product's tables" + @echo " (DESTRUCTIVE; captures to a dump first)" @echo "" @echo "Cleanup:" @echo " make clean Remove build artifacts and cache" @@ -96,12 +91,11 @@ pre-commit: check: lint typecheck test @echo "All checks passed." -run-brief: - insightweaver brief - -run-forecast: - insightweaver forecast - +# run-brief / run-forecast / db-migrate / db-migrate-down / db-reset were +# removed on 2026-08-31 (backlog task 012): every command and migration they +# invoked was deleted with the briefing product. db-reset in particular ran +# `rm -f data/insightweaver.db` behind a y/N prompt, which is not a guard worth +# keeping pointed at a 55,249-row corpus. clean: find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true find . -type d -name "*.egg-info" -exec rm -rf {} + 2>/dev/null || true @@ -119,18 +113,9 @@ clean-all: clean rm -rf venv/ rm -rf .venv/ -db-migrate: - python -m src.database.migrations.add_forecast_tables - -db-migrate-down: - python -m src.database.migrations.add_forecast_tables down - -db-reset: - @echo "WARNING: This will delete all data!" - @read -p "Are you sure? [y/N] " confirm; \ - if [ "$$confirm" = "y" ] || [ "$$confirm" = "Y" ]; then \ - rm -f data/insightweaver.db; \ - python -m src.database.migrations.add_forecast_tables; \ - else \ - echo "Cancelled"; \ - fi +# Deliberately does NOT pass --confirm. The migration refuses without it, so +# running this target prints what it would destroy and stops; the operator types +# the real command themselves. A Makefile target that drops tables on `make` is +# the accident this arrangement exists to prevent. +db-drop-briefing: + $(PYTHON) -m src.database.migrations.drop_briefing_tables diff --git a/README.md b/README.md index efb99ab..18e6ebd 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,24 @@ # InsightWeaver +> **Superseded 2026-08-31 (backlog task 012).** The briefing product this +> document describes -- `brief`, beats, questions, predictions, frames, +> synthesis and rendering -- has been **deleted**. Roughly 8,900 lines of source +> and most of the test suite went with it, in a single commit, because the +> operator's call was that git history is the rollback path and dead code in the +> tree reads as current. `insightweaver` now exposes one command, `sources`. +> +> What survives is ingestion: `src/sources/`, `src/rss/`, `src/processors/`, the +> feed config, and five modules ported for tiers not yet built +> (`src/matching/entity_matcher.py`, `src/matching/coverage_probe.py`, +> `src/llm/claude_client.py`, `src/utils/cadence.py`, +> `src/processors/deduplicator.py`). +> +> **Everything below this banner describes the deleted product and has not been +> rewritten.** Rewriting it needs the new pipeline to exist first, which task 012 +> explicitly puts out of scope. Read it as history until it is replaced. + +--- + **A tool for building warranted, inspectable understanding from RSS feeds.** --- diff --git a/config/beats/us-public-sector-compliance.json b/config/beats/us-public-sector-compliance.json deleted file mode 100644 index ef6193a..0000000 --- a/config/beats/us-public-sector-compliance.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "name": "us-public-sector-compliance", - "description": "Regulatory, legislative, judicial and federal cybersecurity activity that changes what US public sector bodies and their contractors are obliged to do.", - "sources": [ - { - "adapter": "rss", - "feed_tags": ["regulatory", "federal_policy", "legislative", "judicial"], - "geo_tags": ["usa"] - }, - { - "adapter": "rss", - "feed_tags": ["cybersecurity"], - "geo_tags": ["usa"] - }, - { - "adapter": "federal_register", - "feed_tags": ["regulatory", "federal_policy", "cybersecurity"], - "geo_tags": ["usa"] - } - ], - "coverage": { - "orgs": [ - { "name": "FedRAMP PMO", "aliases": ["FedRAMP Program Management Office"] }, - { "name": "CISA", "aliases": ["Cybersecurity and Infrastructure Security Agency"] }, - { "name": "OMB", "aliases": ["Office of Management and Budget"] }, - { "name": "GSA", "aliases": ["General Services Administration"] }, - { "name": "NIST", "aliases": ["National Institute of Standards and Technology"] }, - { "name": "DoD CIO", "aliases": ["DOD CIO", "Department of Defense CIO"] }, - { "name": "DISA", "aliases": ["Defense Information Systems Agency"] } - ], - "programs": [ - { "name": "FedRAMP", "aliases": ["Federal Risk and Authorization Management Program"] }, - { "name": "FedRAMP 20x" }, - { "name": "CMMC", "aliases": ["Cybersecurity Maturity Model Certification"] }, - { "name": "StateRAMP" }, - { "name": "TX-RAMP" } - ], - "document_types": [ - { "name": "Binding Operational Directive", "aliases": ["BOD"] }, - { "name": "Emergency Directive" }, - { "name": "OMB Memorandum", "aliases": ["OMB memo"] }, - { "name": "Security Requirements Guide", "aliases": ["SRG"] }, - { "name": "Notice of Proposed Rulemaking", "aliases": ["NPRM"] } - ] - }, - "standing_questions": [ - "Does CMMC Phase 2 slip past its statutory date?", - "Which CSPs move to FedRAMP authorized, and at which impact level?", - "Does any CISA BOD create a compliance obligation with a deadline inside 90 days?", - "Where do GovRAMP and TX-RAMP diverge from FedRAMP in ways that matter to a multi-state CSP?" - ], - "coverage_probes": [ - { - "date": "2026-08-25", - "what": "OPM issued corrections to its final rules on reduction-in-force, suitability and probationary appeals", - "terms": ["OPM"], - "any_of": [ - ["reduction in force", "suitability", "probationary"], - ["final rule", "correction"] - ] - }, - { - "date": "2026-08-24", - "what": "Supreme Court let the executive order restricting mail-in voting take effect", - "terms": ["Supreme Court", "mail-in voting"], - "any_of": [["allow*", "restriction*", "sides with", "stay"]] - }, - { - "date": "2026-08-24", - "what": "FedRAMP director reinstated", - "terms": ["FedRAMP"], - "any_of": [ - ["director", "administrator"], - ["reinstat*", "return*", "restored"] - ] - }, - { - "date": "2021-11-03", - "what": "CISA issued Binding Operational Directive 22-01, creating the Known Exploited Vulnerabilities catalog and its remediation deadlines", - "terms": ["CISA"], - "any_of": [ - ["binding operational directive", "BOD 22-01"], - ["known exploited", "catalog"] - ] - } - ], - "channels": ["terminal"] -} diff --git a/config/probes/known-events.json b/config/probes/known-events.json new file mode 100644 index 0000000..2c5fbd5 --- /dev/null +++ b/config/probes/known-events.json @@ -0,0 +1,74 @@ +{ + "_provenance": "Authored by hand for the us-public-sector-compliance beat and recovered from commit 673a8dd when backlog task 012 deleted that beat. Beats no longer exist; these are kept because they are real, verified past events in the operator's domain and task 018's staleness check needs exactly this shape. The loader for the old beat format is gone -- task 018 reads this file directly. Recovered 2026-08-31.", + "probes": [ + { + "date": "2026-08-25", + "what": "OPM issued corrections to its final rules on reduction-in-force, suitability and probationary appeals", + "terms": [ + "OPM" + ], + "any_of": [ + [ + "reduction in force", + "suitability", + "probationary" + ], + [ + "final rule", + "correction" + ] + ] + }, + { + "date": "2026-08-24", + "what": "Supreme Court let the executive order restricting mail-in voting take effect", + "terms": [ + "Supreme Court", + "mail-in voting" + ], + "any_of": [ + [ + "allow*", + "restriction*", + "sides with", + "stay" + ] + ] + }, + { + "date": "2026-08-24", + "what": "FedRAMP director reinstated", + "terms": [ + "FedRAMP" + ], + "any_of": [ + [ + "director", + "administrator" + ], + [ + "reinstat*", + "return*", + "restored" + ] + ] + }, + { + "date": "2021-11-03", + "what": "CISA issued Binding Operational Directive 22-01, creating the Known Exploited Vulnerabilities catalog and its remediation deadlines", + "terms": [ + "CISA" + ], + "any_of": [ + [ + "binding operational directive", + "BOD 22-01" + ], + [ + "known exploited", + "catalog" + ] + ] + } + ] +} diff --git a/pyproject.toml b/pyproject.toml index b2dcb86..514cf4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -173,14 +173,18 @@ module = [ ] ignore_errors = true +# Updated 2026-08-31 (backlog task 012). `src.context.*`, `src.pipeline.*` and +# `src.prompts.*` were deleted; `src.forecast.*` had not existed for some time. +# `src.matching.*` and `src.llm.*` are the new homes of the five modules ported +# out of `src.context`, and they inherit the suppression those files already had +# rather than gaining a new one -- the annotations here are SQLAlchemy Column +# descriptors mypy reads as their column type, which predates this task. [[tool.mypy.overrides]] module = [ "src.processors.*", "src.rss.*", - "src.context.*", - "src.forecast.*", - "src.pipeline.*", - "src.prompts.*", + "src.matching.*", + "src.llm.*", "src.feed_manager", ] ignore_errors = true diff --git a/src/cli/app.py b/src/cli/app.py index 2674748..d828060 100644 --- a/src/cli/app.py +++ b/src/cli/app.py @@ -1,6 +1,14 @@ """ -InsightWeaver CLI Application -Click-based multi-command interface +InsightWeaver CLI application. + +Backlog task 012 deleted the briefing product, and with it every command that +generated, rendered, or reasoned over a brief: brief, frames, diet, questions, +predictions, forecast, decisions, beat, stake and scope. What is left is the +source layer, which the rewrite keeps. + +The command table is deliberately short. It is not a placeholder for the new +pipeline -- building that is a separate task -- and nothing here should grow a +command back by habit. """ import contextlib @@ -9,154 +17,37 @@ import click -from .beat import beat_command -from .brief import brief_group from .colors import accent, header, muted -from .decisions import decisions_command -from .diet import diet_command -from .forecast import forecast_command -from .frames import frames_command from .output import set_debug_mode -from .predictions import predictions_command -from .questions import questions_command from .sources import sources_command -from .stake import predict_command, resolve_command # Maps command prefix to its Click command object. -# Order matters: longer prefixes checked first via startswith(). -# In particular "predictions" must precede "predict", since the shorter name is -# a prefix of the longer one (2026-08-27, backlog task 011). +# +# Order used to matter here: "predictions" had to precede "predict", since the +# shorter name is a prefix of the longer one (2026-08-27, backlog task 011). +# Both commands are gone, but the startswith() dispatch below is unchanged, so +# the ordering constraint still applies to whatever is added next. COMMAND_DISPATCH = { - "beat": beat_command, - "brief": brief_group, - "forecast": forecast_command, - "frames": frames_command, - "questions": questions_command, - "predictions": predictions_command, - "predict": predict_command, - "resolve": resolve_command, - "decisions": decisions_command, - "diet": diet_command, "sources": sources_command, } def print_command_refresher(): """Print a short refresher of available commands.""" - refresher = ( - f"\n{header('Commands:')} {accent('brief')} | {accent('beat')} | {accent('forecast')} | " - f"{accent('frames')} | {accent('questions')} | {accent('predictions')} | " - f"{accent('predict')} | {accent('resolve')} | " - f"{accent('decisions')} | {accent('diet')} | {accent('sources')} | " - f"help | exit\n" - ) - click.echo(refresher) + click.echo(f"\n{header('Commands:')} {accent('sources')} | help | exit\n") def print_help(): """Print full help text for interactive mode.""" click.echo(header("Available commands:")) - click.echo(f" {accent('brief')} - Generate intelligence brief and report") - click.echo(f" {accent('beat')} - Test a beat's sources against real events") - click.echo( - f" {accent('forecast')} - View open observables and resolved track record" - ) - click.echo(f" {accent('frames')} - Manage narrative frame glossary") - click.echo(f" {accent('questions')} - Inspect the persistent question graph") - click.echo(f" {accent('predictions')} - Inspect the predictions ledger") - click.echo(f" {accent('predict')} - Stake your own claim on a question") - click.echo(f" {accent('resolve')} - Record the outcome of a prediction") - click.echo(f" {accent('decisions')} - Manage the decision journal") - click.echo(f" {accent('diet')} - Inspect your information diet's structure") - click.echo(f" {accent('sources')} - Per-source structural calibration") + click.echo(f" {accent('sources')} - Inspect configured feeds and what they hold") click.echo(f" {accent('help')} - Show this help message") click.echo(f" {accent('exit')} - Exit InsightWeaver") click.echo() - click.echo(header("Brief command options:")) - click.echo(f" {accent('--hours N')} - Look back N hours (default: 24)") - click.echo(f" {accent('--save PATH')} - Save brief as markdown to PATH") - click.echo(f" {accent('--quiet')} - Compact output (titles only)") - click.echo(muted(" Topic filters: --cybersecurity (-cs), --ai (-ai)")) - click.echo( - muted(" Scope filters: --local (-l), --state (-s), --national (-n), --global (-g)") - ) - click.echo() - click.echo(header("Scope a brief to a subject instead of to you:")) - click.echo(f" {accent('--beat NAME')} - Use config/beats/NAME.json for sources") - click.echo() - click.echo(header("Re-render a stored brief (no pipeline, no API key):")) - click.echo(f" {accent('--from-run ID')} - Render stored brief ID instead of running") - click.echo(f" {accent('--format')} - terminal (default), html, or email") - click.echo(f" {accent('--output PATH')} - Destination file for --format html") - click.echo() - click.echo(header("Forecast command options:")) - click.echo(f" {accent('--days N')} - Resolved-record window (default: 60)") - click.echo(f" {accent('--due')} - What is due right now, at each question's") - click.echo(muted(" own cadence. Stamps what it surfaces.")) - click.echo() - click.echo(header("Frames command:")) - click.echo(f" {accent('frames list')} - List all topic clusters and frames") - click.echo(f" {accent('frames show')} - Show frames for a topic") - click.echo(f" {accent('frames edit')} - Edit a frame in $EDITOR") - click.echo(f" {accent('frames gaps')} - Show recurring perspective gaps") - click.echo() - click.echo(header("Questions command:")) - click.echo(f" {accent('questions list')} - List questions, cadence, next review") - click.echo(muted(" Add --beat NAME to any ledger view to read that beat's ledger")) - click.echo(f" {accent('questions add')} '...' --cadence 90d - Declare a question you carry") - click.echo(f" {accent('questions show')} - Show a question's full history") - click.echo(f" {accent('questions resolve')} --note '...' - Resolve a question") - click.echo() - click.echo(header("Predictions command:")) - click.echo(f" {accent('predictions open')} - Predictions still waiting on coverage") - click.echo(f" {accent('predictions triggered')} - Predictions later coverage confirmed") - click.echo(f" {accent('predictions contradicted')} - Predictions later coverage went against") - click.echo(f" {accent('predictions track-record')} - Your hit rate; model shown separately") - click.echo() - click.echo(header("Stake and resolve your own claims:")) - click.echo(f" {accent('predict')} '...' --by YYYY-MM-DD --confidence 0.7") - click.echo(muted(" Both flags required; a claim with no date is refused, not stored")) - click.echo(f" {accent('resolve')} --outcome yes|no --note '...'") - click.echo(muted(" You grade your own calls. Nothing grades them for you")) - click.echo() - click.echo(header("Decisions command:")) - click.echo(f" {accent('decisions list')} - List standing decisions") - click.echo(f" {accent('decisions show')} - Show factors and routed evidence") - click.echo(f" {accent('decisions add')} --name '...' --type ... - Add a decision") - click.echo(f" {accent('decisions factor add')} --name '...' - Add a factor") - click.echo(f" {accent('decisions resolve')} --note '...' - Mark a decision decided") - click.echo() - click.echo(header("Diet command:")) - click.echo(f" {accent('diet feeds')} - Per-feed frame fingerprint") - click.echo(f" {accent('diet gaps')} - Recurring frame absences") - click.echo(f" {accent('diet overlap')} - Which frames only one feed carries") - click.echo() - click.echo(header("Beat command:")) - click.echo(f" {accent('beat coverage')} - Can this beat see events that happened?") - click.echo(muted(" Reads config/beats/NAME.json coverage_probes. No API key, no writes.")) - click.echo(muted(" Exits 1 if a probe is unmatched, 2 if nothing could be measured.")) - click.echo() click.echo(header("Sources command:")) - click.echo(f" {accent('sources list')} - All feeds with calibration signals") + click.echo(f" {accent('sources list')} - All feeds with stored-article counts") click.echo(f" {accent('sources show')} - Detailed view for one feed") click.echo() - click.echo(header("Examples:")) - click.echo(muted(" brief (24-hour brief, all topics)")) - click.echo(muted(" brief -cs -n (national cybersecurity news)")) - click.echo(muted(" brief --hours 48 -l (48-hour local news brief)")) - click.echo(muted(" brief --save brief.md (save brief as markdown)")) - click.echo(muted(" brief --from-run 176 (re-render stored brief 176)")) - click.echo(muted(" brief --beat us-public-sector-compliance (subject brief)")) - click.echo(muted(" beat coverage us-public-sector-compliance (can it see real events?)")) - click.echo(muted(" forecast (open observables + last 60d resolved)")) - click.echo(muted(" forecast --days 30 (tighter resolved-record window)")) - click.echo(muted(" forecast --due (what is due now, at each question's cadence)")) - click.echo(muted(" questions add 'Does CMMC Phase 2 slip?' --cadence 90d")) - click.echo(muted(" predict 23 'Yes -- slips' --by 2026-12-31 --confidence 0.7")) - click.echo(muted(" resolve 41 --outcome no --note 'class deviation published'")) - click.echo(muted(" frames list (view narrative frame glossary)")) - click.echo(muted(" frames gaps (view perspective gaps in your feeds)")) - click.echo() click.echo(muted("Tip: Add --debug to any command to see detailed logs")) click.echo() @@ -167,11 +58,9 @@ def _dispatch_command(command: str) -> bool: Returns True if a command was matched, False otherwise. - Argument splitting is quote-aware (shlex) as of 2026-08-27: `predict 23 - "Yes -- slips" --by ...` takes a multi-word claim as one argument, and a - plain whitespace split would have handed Click four arguments instead of - one. The same applies to every option that takes prose, such as - `resolve --note`. + Argument splitting is quote-aware (shlex) as of 2026-08-27: an option that + takes prose needs its value delivered as one argument, and a plain + whitespace split would hand Click several instead. """ for prefix, click_cmd in COMMAND_DISPATCH.items(): if command.startswith(prefix): @@ -187,9 +76,8 @@ def _dispatch_command(command: str) -> bool: try: click_cmd.main(args, standalone_mode=False) except click.ClickException as exc: - # standalone_mode=False re-raises instead of printing. - # The write commands reject bad input this way, so the - # rejection has to reach the operator intact. + # standalone_mode=False re-raises instead of printing, so + # the rejection has to be shown here to reach the operator. exc.show() print_command_refresher() return True @@ -266,10 +154,10 @@ def interactive_mode(): @click.version_option(version="1.0.0", prog_name="InsightWeaver") def cli(ctx, debug): """ - InsightWeaver - Intelligent RSS Feed Analysis System + InsightWeaver - RSS feed ingestion and source inspection. - Transform RSS feed data streams into coherent, actionable narratives - through location-specific, integrated perspectives. + The briefing product was removed in backlog task 012. What remains is + ingestion and the source layer. """ set_debug_mode(debug) ctx.ensure_object(dict) diff --git a/src/cli/ascii_art.py b/src/cli/ascii_art.py deleted file mode 100644 index 4fcdbee..0000000 --- a/src/cli/ascii_art.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -ASCII art for CLI commands -Hacker aesthetic visual identifiers for each command -""" - -import sys - -from .colors import accent, header - - -def _is_tty() -> bool: - """Check if running in an interactive terminal""" - return sys.stdout.isatty() - - -def render_brief_art(status: str = "Analyzing the signal...") -> str: - """ - Render ASCII art for the brief command - Intelligence Analyst figure - - Args: - status: Status message to display next to the art - - Returns: - Formatted ASCII art string with colors - """ - title = "INTELLIGENCE BRIEF" - underline = "=" * len(title) - - # Build art line by line to avoid escaping issues - lines = [ - "", - " " + accent("___"), - " " + accent("/ \\"), - " " + accent("| O O |"), - " " + accent("\\ - /") + " " + header(title), - " " + accent("/| |\\") + " " + header(underline), - " " + accent("/ | | \\") + " " + status, - " " + accent("/ |___| \\"), - "", - ] - return "\n".join(lines) - - -def render_forecast_art(status: str = "Gazing into the future...") -> str: - """ - Render ASCII art for the forecast command - Crystal Ball - - Args: - status: Status message to display next to the art - - Returns: - Formatted ASCII art string with colors - """ - title = "FORECAST" - underline = "=" * len(title) - - lines = [ - "", - " " + accent(".-."), - " " + accent("( )"), - " " + accent("/` `\\") + " " + header(title), - " " + accent("| ___ |") + " " + header(underline), - " " + accent("| ( ) |") + " " + status, - " " + accent("\\`---'/"), - " " + accent("`---'"), - "", - ] - return "\n".join(lines) - - -def should_show_art(quiet: bool = False) -> bool: - """ - Determine if ASCII art should be displayed - - Args: - quiet: User-specified quiet mode flag - - Returns: - True if art should be shown, False otherwise - """ - if quiet: - return False - return _is_tty() diff --git a/src/cli/beat.py b/src/cli/beat.py deleted file mode 100644 index 021b210..0000000 --- a/src/cli/beat.py +++ /dev/null @@ -1,221 +0,0 @@ -""" -Beat Command - test a beat's sources against events that actually happened. - -``insightweaver beat coverage NAME`` runs the beat's declared -``coverage_probes`` against the stored corpus and reports, per probe, whether an -article the beat's own sources carried would have shown the operator the event, -and which source that was. - -It exits non-zero when a probe went unmatched, so it can gate a release the way -a test does. It also exits non-zero when *nothing was measured* -- no probes -declared, or every probe inconclusive -- because "no evidence of a gap" and "no -evidence" are the same green tick and only one of them is good news. That -conflation is the whole reason backlog task 010 exists. - -No model is called and nothing is written. Added 2026-08-27. -""" - -import click - -from ..config.beats import BeatNotFound, BeatValidationError, load_beat -from ..context.coverage_probe import ( - STATUS_INCONCLUSIVE, - STATUS_MATCHED, - STATUS_UNMATCHED, - CoverageReport, - ProbeResult, - run_coverage_probes, -) -from ..database.connection import SessionLocal -from .colors import accent, error, header, muted, success, warning - -# Exit codes. Distinct on purpose: a gate that fails and a gate that could not -# run are different alarms, and collapsing them would let a rotted probe set -# read as a broken beat (or, worse, the reverse). -EXIT_OK = 0 -EXIT_UNMATCHED = 1 -EXIT_NOTHING_MEASURED = 2 - -_STATUS_STYLE = { - STATUS_MATCHED: success, - STATUS_UNMATCHED: error, - STATUS_INCONCLUSIVE: warning, -} - - -@click.group(name="beat") -def beat_command(): - """Inspect a beat's configuration against the corpus.""" - pass - - -@beat_command.command(name="coverage") -@click.argument("name") -@click.option( - "--show", - type=click.Choice(["all", "problems"]), - default="all", - help="'problems' prints only the probes that did not pass.", -) -def coverage(name, show): - """ - Check whether this beat's sources can see events that actually happened. - - Each probe in the beat's `coverage_probes` names a real past event and the - words any report of it would carry. A probe passes only when a feed this - beat subscribes to carried a matching article inside the probe's window. - - Exits 1 if any probe is unmatched, 2 if nothing could be measured. - """ - try: - beat = load_beat(name) - except (BeatNotFound, BeatValidationError) as exc: - raise click.ClickException(str(exc)) - - if not beat.coverage_probes: - click.echo(error(f"Beat '{beat.name}' declares no coverage_probes.")) - click.echo( - muted( - "Nothing was measured. An article count tells you ingestion is running;\n" - "only a probe tells you it reaches this domain. Name three things that\n" - "actually happened in this beat's subject this month and declare them in\n" - f"{beat.config_path} under 'coverage_probes'." - ) - ) - raise SystemExit(EXIT_NOTHING_MEASURED) - - # Read-only by construction: this session is never committed, because the - # corpus is shared and a question about it must not alter it. - session = SessionLocal() - try: - report = run_coverage_probes(session, beat) - finally: - session.rollback() - session.close() - - _print_report(report, beat.config_path, show) - code = _exit_code(report) - if code != EXIT_OK: - raise SystemExit(code) - - -def _exit_code(report: CoverageReport) -> int: - """ - The verdict, as a shell sees it. - - ``EXIT_NOTHING_MEASURED`` for an all-inconclusive run is the load-bearing - case: with no probe answered there is no evidence the beat reaches anything, - and exiting 0 there would hand a gate the same green tick a real pass gets. - """ - if report.unmatched: - return EXIT_UNMATCHED - if not report.conclusive: - return EXIT_NOTHING_MEASURED - return EXIT_OK - - -def _print_report(report: CoverageReport, config_path: str, show: str) -> None: - click.echo(header(f"COVERAGE PROBES - beat '{report.beat_name}'")) - click.echo("=" * 80) - span = ( - f"{report.earliest.date().isoformat()} to {report.latest.date().isoformat()}" - if report.earliest and report.latest - else "no dated articles" - ) - click.echo(muted(f" sources: {len(report.feed_names)} feeds | corpus span: {span}")) - click.echo(muted(f" probes declared in {config_path}")) - click.echo() - - for result in report.results: - if show == "problems" and result.matched: - continue - _print_result(result) - - _print_summary(report) - - -def _print_result(result: ProbeResult) -> None: - style = _STATUS_STYLE[result.status] - start, end = result.window - click.echo(f"{style(f'[{result.status}]')} {accent(result.probe.what)}") - click.echo(muted(f" event {result.probe.date.isoformat()} | window {start} .. {end}")) - click.echo(muted(f" requires {result.probe.describe()}")) - click.echo(muted(f" {result.articles_in_window} article(s) from this beat's feeds in window")) - - if result.status == STATUS_MATCHED: - click.echo(muted(f" {result.match_count} matching article(s); earliest shown")) - for match in result.matches: - click.echo( - f" {success('seen by')} {accent(match.feed_name)} " - f"{muted(f'({match.published_day})')}" - ) - click.echo(muted(f" {match.title}")) - elif result.status == STATUS_UNMATCHED: - click.echo(error(" no article from this beat's feeds matched")) - if result.missing_terms: - click.echo(muted(f" closest article lacked: {', '.join(result.missing_terms)}")) - for group in result.unsatisfied_groups: - click.echo(muted(f" closest article lacked any of: {', '.join(group)}")) - if result.elsewhere: - click.echo( - warning( - f" but {result.elsewhere_count} article(s) elsewhere in the corpus " - f"matched, outside this beat's sources:" - ) - ) - for match in result.elsewhere: - click.echo(f" {accent(match.feed_name)} {muted(f'({match.published_day})')}") - click.echo(muted(f" {match.title}")) - click.echo(muted(" the sources are reachable; this beat's list is what missed.")) - else: - click.echo(muted(" and no feed anywhere in the corpus carried it either.")) - else: - click.echo(warning(f" {result.reason}")) - click.echo(muted(" neither a pass nor a failure; still counted below.")) - click.echo() - - -def _print_summary(report: CoverageReport) -> None: - """ - The counts, with the denominator always the number of probes declared. - - Printed as `matched + unmatched + inconclusive of N declared` so the - arithmetic is visible. A probe set decaying out of retention has to show up - here as a shrinking number of *answered* probes rather than as a clean sweep - of the few that still match. - """ - click.echo("=" * 80) - matched, unmatched, inconclusive = ( - len(report.matched), - len(report.unmatched), - len(report.inconclusive), - ) - click.echo( - f"{success(f'{matched} matched')} + {error(f'{unmatched} unmatched')} + " - f"{warning(f'{inconclusive} inconclusive')} of {report.total} probe(s) declared" - ) - - if unmatched: - click.echo( - error( - f"FAIL: {unmatched} event(s) this beat claims to cover left no trace " - f"in its sources." - ) - ) - elif not report.conclusive: - click.echo( - warning( - "NOT MEASURED: every probe was inconclusive. This is not a pass -- the " - "corpus could not answer a single one of them." - ) - ) - else: - click.echo(success(f"PASS: {matched} of {report.total} probe(s) answered, none missed.")) - - if inconclusive and not unmatched and report.conclusive: - click.echo( - muted( - f" {inconclusive} probe(s) could not be answered and are counted above, " - f"not dropped." - ) - ) diff --git a/src/cli/brief.py b/src/cli/brief.py deleted file mode 100644 index e7d9cf8..0000000 --- a/src/cli/brief.py +++ /dev/null @@ -1,455 +0,0 @@ -""" -Brief Command - Intelligence Brief Generation -Preserves all existing InsightWeaver functionality -""" - -import asyncio -import logging -from pathlib import Path - -import click - -from ..config.beats import BeatConfig, BeatNotFound, BeatValidationError, load_beat -from ..config.settings import settings -from ..context.beat_scope import BeatTablesMissing, require_beat_tables -from ..database.connection import create_tables, engine -from ..feed_manager import setup_feeds -from ..pipeline.orchestrator import run_pipeline -from ..render.document import BriefDocument, StoredBriefNotFound, load_stored_brief -from ..render.email import EmailDeliveryError, EmailRenderer -from ..render.html import HTMLRenderer -from ..render.markdown import MarkdownRenderer -from ..render.terminal import TerminalRenderer -from .colors import muted, warning -from .loading import loading -from .output import get_output_manager, is_debug_mode - -logger = logging.getLogger(__name__) - -# Output media a stored brief can be rendered to. The live pipeline path is -# unchanged and always renders to the terminal. -RENDER_FORMATS = ("terminal", "html", "email") - - -# ============================================================================ -# Helper Functions (from main.py) -# ============================================================================ - - -async def run_fetch_only(): - """Run only RSS fetching""" - from ..rss.parallel_fetcher import fetch_all_active_feeds - - print("Running RSS feed fetching...") - results = await fetch_all_active_feeds() - print( - f"Fetched {results['total_articles']} articles from {results['successful_feeds']}/{results['total_feeds']} feeds" - ) - return results - - -def setup_database(): - """Initialize database and load feeds""" - print("Setting up database...") - create_tables() - print("Database tables created") - - print("Loading RSS feeds...") - fm = setup_feeds() - stats = fm.get_feed_statistics() - print( - f"Loaded {stats['database']['active_feeds']} active feeds across {len(stats['database']['categories'])} categories" - ) - - -def load_beat_or_fail(beat_name: str) -> BeatConfig: - """ - Load a beat by name, turning both failure modes into CLI errors. - - A misnamed beat lists what is available; a malformed one names the problem. - Neither is silently tolerated: running a beat brief against a half-understood - definition would produce a brief nobody can account for. - """ - try: - beat = load_beat(beat_name) - except (BeatNotFound, BeatValidationError) as exc: - raise click.ClickException(str(exc)) - - # A beat run must be recordable before it starts; see require_beat_tables. - try: - require_beat_tables(engine) - except BeatTablesMissing as exc: - raise click.ClickException(str(exc)) - - return beat - - -# ============================================================================ -# Render-only path (--from-run) -# ============================================================================ - - -def default_html_path(doc: BriefDocument) -> Path: - """Where ``--format html`` writes when ``--output`` is not given.""" - return settings.data_dir / "briefs" / f"brief-{doc.synthesis_id}.html" - - -def render_stored_brief( - synthesis_id: int, - output_format: str, - output_path: str | None, - quiet: bool, - save_path: str | None, -) -> None: - """ - Render a stored ``narrative_syntheses`` row. - - This replays what a past run produced; it never fetches, synthesizes or - calls Claude, so it needs no API key and no network, and rendering the same - id twice produces the same bytes. - """ - try: - doc = load_stored_brief(synthesis_id) - except StoredBriefNotFound as exc: - raise click.ClickException(str(exc)) - - if output_format == "terminal": - renderer = TerminalRenderer() - click.echo(renderer.render_compact(doc) if quiet else renderer.render(doc)) - elif output_format == "html": - target = Path(output_path) if output_path else default_html_path(doc) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(HTMLRenderer().render(doc), encoding="utf-8") - click.echo(muted(f"Wrote HTML brief to {target}")) - elif output_format == "email": - try: - recipient = EmailRenderer().send(doc) - except EmailDeliveryError as exc: - raise click.ClickException(str(exc)) - click.echo(muted(f"Sent brief {synthesis_id} to {recipient}")) - - if save_path: - Path(save_path).write_text(MarkdownRenderer().render(doc)) - click.echo(muted(f"Saved markdown to {save_path}")) - - -# ============================================================================ -# Click Command Group -# ============================================================================ - - -@click.group(invoke_without_command=True) -@click.pass_context -@click.option("--hours", type=int, default=24, help="Look back N hours for analysis (default: 24)") -@click.option( - "--quiet", - "-q", - is_flag=True, - help="Compact output (situation titles only, for scripts/automation)", -) -@click.option( - "--cybersecurity", - "-cs", - "filter_cybersecurity", - is_flag=True, - help="Filter to cybersecurity-related articles only", -) -@click.option( - "--ai", "-ai", "filter_ai", is_flag=True, help="Filter to AI/ML-related articles only" -) -@click.option("--local", "-l", "filter_local", is_flag=True, help="Filter to local news only") -@click.option( - "--state", "-s", "filter_state", is_flag=True, help="Filter to state/regional news only" -) -@click.option( - "--national", "-n", "filter_national", is_flag=True, help="Filter to national news only" -) -@click.option( - "--global", "-g", "filter_global", is_flag=True, help="Filter to global/international news only" -) -@click.option( - "--save", - "save_path", - type=click.Path(dir_okay=False, writable=True), - default=None, - help="Save brief as markdown to PATH", -) -@click.option( - "--from-run", - "from_run", - type=int, - default=None, - metavar="ID", - help="Render a stored brief by narrative_syntheses id instead of running the pipeline", -) -@click.option( - "--format", - "output_format", - type=click.Choice(RENDER_FORMATS), - default="terminal", - help="Output medium for --from-run (default: terminal)", -) -@click.option( - "--output", - "output_path", - type=click.Path(dir_okay=False, writable=True), - default=None, - help="Destination file for --format html", -) -@click.option( - "--beat", - "beat_name", - default=None, - metavar="NAME", - help="Scope the brief to config/beats/NAME.json instead of the user profile", -) -def brief_group( - ctx, - hours, - quiet, - filter_cybersecurity, - filter_ai, - filter_local, - filter_state, - filter_national, - filter_global, - save_path, - from_run, - output_format, - output_path, - beat_name, -): - """ - Run intelligence brief pipeline and generate report - - Fetches RSS feeds, analyzes content with Claude, and generates - intelligence reports tailored to your location and interests. - - You can filter by topic (e.g., --cybersecurity) and/or scope (e.g., --local). - Multiple filters combine with AND logic: --cybersecurity --local shows only local cybersecurity news. - - Pass --from-run ID to re-render a brief already stored in narrative_syntheses. - That path is offline and deterministic: no feeds, no Claude call, no API key, - and the same id always renders the same bytes. - - Pass --beat NAME to run the brief for a subject rather than for you: article - selection is restricted to that beat's configured sources, and the questions - and predictions it accumulates are kept separate from the default brief's. - """ - if ctx.invoked_subcommand is None: - # Render-only path. Deliberately ahead of the API-key check: replaying - # a stored run must work with ANTHROPIC_API_KEY unset. - if from_run is not None: - if beat_name: - raise click.ClickException( - "--beat applies to the live pipeline. --from-run replays a stored " - "run exactly as it was recorded, including the beat it ran for." - ) - render_stored_brief(from_run, output_format, output_path, quiet, save_path) - return - - # Resolve the beat before anything expensive happens: a typo in the - # name should cost nothing. - beat = load_beat_or_fail(beat_name) if beat_name else None - - if output_format != "terminal": - raise click.ClickException( - f"--format {output_format} needs --from-run ID. The live pipeline path " - "renders to the terminal only." - ) - if output_path: - raise click.ClickException("--output only applies to --from-run --format html.") - - # Check for API key - if not settings.anthropic_api_key: - click.echo(warning("Warning: ANTHROPIC_API_KEY not configured")) - click.echo(muted("Brief generation requires Claude API access")) - raise click.Abort() - - # Build topic_filters dict from flags - topic_filters = {} - - # Topic filters (from professional_domains) - topics = [] - if filter_cybersecurity: - topics.append("cybersecurity") - if filter_ai: - topics.append("ai/ml") - - if topics: - topic_filters["topics"] = topics - - # Scope filters (geographic) - scopes = [] - if filter_local: - scopes.append("local") - if filter_state: - scopes.append("state") - if filter_national: - scopes.append("national") - if filter_global: - scopes.append("global") - - if scopes: - topic_filters["scopes"] = scopes - - # Store in context for subcommands - ctx.ensure_object(dict) - ctx.obj["topic_filters"] = topic_filters - - debug = is_debug_mode() - output_mgr = get_output_manager() - - # Build loading message based on filters - if beat: - loading_msg = f"Generating {beat.name} brief" - elif topic_filters: - filter_desc = [] - if topics: - filter_desc.append(f"{', '.join(topics)}") - if scopes: - filter_desc.append(f"{', '.join(scopes)}") - loading_msg = f"Generating {' '.join(filter_desc)} brief" - else: - loading_msg = "Generating intelligence brief" - - # Run pipeline (fetch, deduplicate, filter, synthesize) - async def run_brief(): - pipeline_result = await run_pipeline( - prioritize_hours=hours, topic_filters=topic_filters, beat=beat - ) - - # Extract synthesis result from pipeline - synthesis_stage = pipeline_result.get("stages", {}).get("synthesis", {}) - report_result = { - "success": synthesis_stage.get("status") == "success", - "status": synthesis_stage.get("status"), - "articles_analyzed": synthesis_stage.get("articles_analyzed", 0), - "synthesis_data": synthesis_stage.get("synthesis_data", {}), - "synthesis_id": synthesis_stage.get("synthesis_id"), - } - - return {"pipeline": pipeline_result, "report": report_result} - - with loading(loading_msg, debug=debug), output_mgr.suppress_output(): - result = asyncio.run(run_brief()) - - # Display results - if result: - pipeline_summary = result.get("pipeline", {}).get("summary", {}) - report_result = result.get("report", {}) - - # A source that failed or went quiet is reported before the brief, - # not buried in a log. Added 2026-08-26 for backlog task 005: a - # thin brief with no stated cause is the failure mode this whole - # adapter layer exists to prevent. - source_alerts = pipeline_summary.get("source_alerts") or [] - if source_alerts: - click.echo("\n" + warning("!" * 80)) - click.echo(warning("SOURCE ALERT - THIS BRIEF MAY BE INCOMPLETE")) - click.echo(warning("!" * 80)) - for alert in source_alerts: - click.echo(warning(f" {alert}")) - click.echo("=" * 80) - - # Check if filters resulted in 0 articles - articles_analyzed = report_result.get("articles_analyzed", 0) - articles_fetched = pipeline_summary.get("articles_fetched", 0) - - synthesis_status = report_result.get("status") - - if beat and articles_analyzed == 0 and synthesis_status == "no_articles": - # A beat with no articles is a real and expected outcome while - # RSS is the only adapter -- say so plainly rather than - # printing an empty brief. - click.echo("\n" + warning("!" * 80)) - click.echo(warning(f"NO ARTICLES IN BEAT '{beat.name}'")) - click.echo(warning("!" * 80)) - click.echo( - muted( - f"\nThe beat selects {len(beat.resolve_feeds())} configured feed(s), " - f"but none of them yielded articles in the last {hours} hours." - ) - ) - click.echo("\nSuggestions:") - click.echo(" - Widen the window: --hours 168") - click.echo(" - Check the beat's feeds are loaded: brief setup") - click.echo(" - The domain may simply not publish much RSS") - click.echo("=" * 80) - elif topic_filters and articles_analyzed == 0 and articles_fetched > 0: - # Filters were too restrictive - click.echo("\n" + warning("!" * 80)) - click.echo(warning("NO ARTICLES MATCHED YOUR FILTERS")) - click.echo(warning("!" * 80)) - click.echo(muted(f"\nFetched {articles_fetched} articles, but none matched:")) - if "topics" in topic_filters: - click.echo(f" Topics: {', '.join(topic_filters['topics'])}") - if "scopes" in topic_filters: - click.echo(f" Scopes: {', '.join(topic_filters['scopes'])}") - click.echo("\nSuggestions:") - click.echo(" • Try expanding the time window: --hours 48 or --hours 168") - click.echo(" • Use fewer filters (remove -s or -cs)") - click.echo(" • Try different filter combinations") - click.echo(" • Run without filters to see all available content") - click.echo("=" * 80) - elif report_result.get("success") and articles_analyzed > 0: - # Same document model and same renderers the --from-run path - # uses; the only difference is where the document came from. - doc = BriefDocument.from_report(report_result) - renderer = TerminalRenderer() - - if quiet: - click.echo(renderer.render_compact(doc)) - else: - click.echo(renderer.render(doc)) - - if save_path: - Path(save_path).write_text(MarkdownRenderer().render(doc)) - click.echo(muted(f"\nSaved markdown to {save_path}")) - - # Pipeline summary - click.echo() - click.echo(muted(f"Duration: {pipeline_summary.get('duration_seconds', 0):.1f}s")) - - # Print profiling report in debug mode - if debug: - from ..utils.profiler import get_profiler - - profiler = get_profiler() - profiler.print_summary() - - # Interactive frame validation (only in TTY, after successful brief) - if ( - report_result.get("success") - and articles_analyzed > 0 - and click.get_text_stream("stdin").isatty() - ): - from .frames import run_frame_validation_loop - - run_frame_validation_loop() - - -# ============================================================================ -# Subcommands -# ============================================================================ - - -@brief_group.command(name="setup") -def setup_cmd(): - """Setup database and load RSS feeds""" - setup_database() - - -@brief_group.command(name="fetch") -def fetch_cmd(): - """Only fetch RSS feeds (no analysis)""" - debug = is_debug_mode() - output_mgr = get_output_manager() - - with loading("Fetching RSS feeds", debug=debug), output_mgr.suppress_output(): - result = asyncio.run(run_fetch_only()) - - if result: - click.echo( - f"\n✓ Fetched {result['total_articles']} articles from {result['successful_feeds']}/{result['total_feeds']} feeds" - ) diff --git a/src/cli/brief_formatter.py b/src/cli/brief_formatter.py deleted file mode 100644 index d75180a..0000000 --- a/src/cli/brief_formatter.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Brief Report Terminal Formatter -- compatibility surface. - -The formatting itself moved to :mod:`src.render`: one document model -(:class:`~src.render.document.BriefDocument`) and one renderer per medium. -Nothing here formats anything; it adapts the legacy ``report_data`` dict shape -onto a ``BriefDocument`` and delegates, so there is exactly one place that -decides what a brief looks like. -""" - -from typing import Any - -from ..render._text import clean_citations -from ..render.document import BriefDocument -from ..render.markdown import MarkdownRenderer -from ..render.terminal import TerminalRenderer - -__all__ = [ - "BriefFormatter", - "clean_citations", -] - - -class BriefFormatter: - """Legacy adapter: takes the pipeline's ``report_data`` dict, delegates.""" - - def __init__(self, max_width: int = 80): - self.max_width = max_width - self._terminal = TerminalRenderer(max_width) - self._markdown = MarkdownRenderer() - - def format_report(self, report_data: dict[str, Any]) -> str: - return self._terminal.render(BriefDocument.from_report(report_data)) - - def format_one_line_summary(self, report_data: dict[str, Any]) -> str: - return self._terminal.render_one_line_summary(BriefDocument.from_report(report_data)) - - def format_markdown(self, report_data: dict[str, Any]) -> str: - return self._markdown.render(BriefDocument.from_report(report_data)) diff --git a/src/cli/decisions.py b/src/cli/decisions.py deleted file mode 100644 index 3ea6efa..0000000 --- a/src/cli/decisions.py +++ /dev/null @@ -1,169 +0,0 @@ -""" -Decisions Command - Manage the decision journal and inspect routed evidence. -""" - -import click - -from ..database.connection import get_db -from ..database.models import ( - DECISION_STATUS_DECIDED, - DECISION_STATUS_OPEN, - Decision, - DecisionEvidence, - DecisionFactor, -) -from ..utils import utcnow -from .colors import accent, header, muted, success, warning - -DECISION_TYPES = ["career", "housing", "education", "financial", "civic", "other"] - - -@click.group(name="decisions") -def decisions_command(): - """Manage standing decisions and the evidence routed into them.""" - pass - - -@decisions_command.command(name="list") -@click.option( - "--status", "-s", type=click.Choice(["open", "decided", "deferred", "all"]), default="open" -) -def list_decisions(status): - """List decisions with factor and evidence counts.""" - with get_db() as session: - query = session.query(Decision) - if status != "all": - query = query.filter(Decision.status == status) - rows = query.order_by(Decision.created_at.asc()).all() - - if not rows: - click.echo(muted(f"No {status} decisions. Add one with 'decisions add'.")) - return - - click.echo(header(f"DECISIONS ({status}, {len(rows)} shown)")) - click.echo("=" * 80) - for d in rows: - factor_count = session.query(DecisionFactor).filter_by(decision_id=d.id).count() - evidence_count = session.query(DecisionEvidence).filter_by(decision_id=d.id).count() - tag = success("open") if d.status == DECISION_STATUS_OPEN else muted(d.status) - click.echo(f"{accent(f'D{d.id}')} [{tag}] {muted(d.decision_type or 'other')}") - click.echo(f" {d.name}") - click.echo(muted(f" {factor_count} factor(s) | {evidence_count} evidence record(s)")) - click.echo() - - -@decisions_command.command(name="show") -@click.argument("decision_id", type=int) -def show_decision(decision_id): - """Show a decision, its factors, and routed evidence per factor.""" - with get_db() as session: - d = session.query(Decision).filter_by(id=decision_id).first() - if not d: - click.echo(warning(f"Decision D{decision_id} not found.")) - return - - click.echo(header(f"DECISION D{d.id}")) - click.echo("=" * 80) - click.echo(d.name) - click.echo() - click.echo(muted(f"Type: {d.decision_type or 'other'} | Status: {d.status}")) - click.echo(muted(f"Created: {d.created_at.date().isoformat()}")) - if d.decided_at: - click.echo(muted(f"Decided: {d.decided_at.date().isoformat()}")) - if d.notes: - click.echo(muted(f"Notes: {d.notes}")) - click.echo() - - factors = session.query(DecisionFactor).filter_by(decision_id=d.id).all() - if not factors: - click.echo(muted("No factors yet. Add one with 'decisions factor add'.")) - return - - for f in factors: - click.echo(header(f"Factor F{f.id}: {f.name}")) - if f.what_would_update_me: - click.echo(muted(f" Would update me: {f.what_would_update_me}")) - evidence = ( - session.query(DecisionEvidence) - .filter_by(factor_id=f.id) - .order_by(DecisionEvidence.observed_at.asc()) - .all() - ) - if not evidence: - click.echo(muted(" No evidence routed yet.")) - for ev in evidence: - status_tag = f" [{ev.epistemic_status}]" if ev.epistemic_status else "" - click.echo( - f" {accent(ev.direction)} {ev.observed_at.date().isoformat()}" - f"{muted(status_tag)}" - ) - click.echo(f" {ev.situation_excerpt}") - click.echo() - - -@decisions_command.command(name="add") -@click.option("--name", required=True, help="What the decision is.") -@click.option( - "--type", - "decision_type", - type=click.Choice(DECISION_TYPES), - default="other", - help="Decision category.", -) -def add_decision(name, decision_type): - """Add a new standing decision.""" - with get_db() as session: - decision = Decision(name=name, decision_type=decision_type) - session.add(decision) - session.commit() - click.echo(success(f"Added decision D{decision.id}: {name}")) - - -@decisions_command.command(name="resolve") -@click.argument("decision_id", type=int) -@click.option("--note", required=True, help="What you decided and why.") -def resolve_decision(decision_id, note): - """Mark a decision decided with a note.""" - with get_db() as session: - d = session.query(Decision).filter_by(id=decision_id).first() - if not d: - click.echo(warning(f"Decision D{decision_id} not found.")) - return - if d.status != DECISION_STATUS_OPEN: - click.echo(warning(f"D{d.id} is already {d.status}; not modifying.")) - return - d.status = DECISION_STATUS_DECIDED - d.decided_at = utcnow() - d.notes = note - session.commit() - click.echo(success(f"D{d.id} marked decided.")) - - -@decisions_command.group(name="factor") -def factor_group(): - """Manage the factors tracked for a decision.""" - pass - - -@factor_group.command(name="add") -@click.argument("decision_id", type=int) -@click.option("--name", required=True, help="What the factor is.") -@click.option( - "--update-when", - "what_would_update_me", - default=None, - help="Your rule for what evidence would change your read.", -) -def add_factor(decision_id, name, what_would_update_me): - """Add a factor to a decision.""" - with get_db() as session: - d = session.query(Decision).filter_by(id=decision_id).first() - if not d: - click.echo(warning(f"Decision D{decision_id} not found.")) - return - factor = DecisionFactor( - decision_id=d.id, name=name, what_would_update_me=what_would_update_me - ) - session.add(factor) - session.commit() - click.echo(success(f"Added factor F{factor.id} to D{d.id}: {name}")) diff --git a/src/cli/diet.py b/src/cli/diet.py deleted file mode 100644 index 46e6753..0000000 --- a/src/cli/diet.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -Diet Command - Surface the editorial structure of your information diet. - -Derived entirely from the article_frames mapping: which frames each feed -carries, which frames are absent, and which frames only one feed supplies. -""" - -import click -from sqlalchemy import func - -from ..database.connection import get_db -from ..database.models import ( - Article, - ArticleFrame, - FrameGap, - NarrativeFrame, - RSSFeed, - TopicCluster, -) -from .colors import accent, header, muted, warning - - -@click.group(name="diet") -def diet_command(): - """Inspect the editorial structure of your feed set.""" - pass - - -@diet_command.command(name="feeds") -def feeds_fingerprint(): - """Per-feed frame fingerprint: which frames each feed carries.""" - with get_db() as session: - rows = ( - session.query( - RSSFeed.name, - NarrativeFrame.label, - func.count(ArticleFrame.id), - ) - .join(Article, ArticleFrame.article_id == Article.id) - .join(RSSFeed, Article.feed_id == RSSFeed.id) - .join(NarrativeFrame, ArticleFrame.frame_id == NarrativeFrame.id) - .group_by(RSSFeed.name, NarrativeFrame.label) - .order_by(RSSFeed.name, func.count(ArticleFrame.id).desc()) - .all() - ) - - if not rows: - click.echo(muted("No frame classifications yet. Run a brief first.")) - return - - by_feed: dict[str, list[tuple[str, int]]] = {} - for feed_name, frame_label, count in rows: - by_feed.setdefault(feed_name, []).append((frame_label, count)) - - click.echo(header("FEED FRAME FINGERPRINTS")) - click.echo("=" * 80) - for feed_name, frames in by_feed.items(): - click.echo(accent(feed_name)) - for frame_label, count in frames: - click.echo(f" {frame_label} {muted(f'({count})')}") - click.echo() - - -@diet_command.command(name="gaps") -def diet_gaps(): - """Recurring frame absences -- a feed curation signal.""" - with get_db() as session: - rows = ( - session.query(FrameGap, TopicCluster.name) - .join(TopicCluster, FrameGap.topic_cluster_id == TopicCluster.id) - .order_by(FrameGap.occurrences.desc()) - .all() - ) - - if not rows: - click.echo(muted("No frame gaps recorded yet.")) - return - - click.echo(header("RECURRING FRAME GAPS")) - click.echo(muted("Frames consistently absent from your feeds.")) - click.echo("=" * 80) - for gap, cluster_name in rows: - click.echo( - f"{warning(gap.frame_label)} {muted(f'({gap.occurrences}x, in {cluster_name})')}" - ) - if gap.feed_suggestion: - click.echo(f" Suggested source: {muted(gap.feed_suggestion)}") - click.echo() - - -@diet_command.command(name="overlap") -def diet_overlap(): - """Which frames only one feed carries, and which are widely carried.""" - with get_db() as session: - rows = ( - session.query( - NarrativeFrame.label, - func.count(func.distinct(RSSFeed.id)), - ) - .join(ArticleFrame, ArticleFrame.frame_id == NarrativeFrame.id) - .join(Article, ArticleFrame.article_id == Article.id) - .join(RSSFeed, Article.feed_id == RSSFeed.id) - .group_by(NarrativeFrame.label) - .order_by(func.count(func.distinct(RSSFeed.id)).asc()) - .all() - ) - - if not rows: - click.echo(muted("No frame classifications yet. Run a brief first.")) - return - - unique = [(label, n) for label, n in rows if n == 1] - shared = [(label, n) for label, n in rows if n > 1] - - click.echo(header("FRAME OVERLAP")) - click.echo("=" * 80) - - click.echo(accent("Carried by only one feed (single point of exposure):")) - if unique: - for label, _ in unique: - click.echo(f" {label}") - else: - click.echo(muted(" none")) - click.echo() - - click.echo(accent("Carried by multiple feeds:")) - if shared: - for label, n in shared: - click.echo(f" {label} {muted(f'({n} feeds)')}") - else: - click.echo(muted(" none")) diff --git a/src/cli/forecast.py b/src/cli/forecast.py deleted file mode 100644 index bcccd7a..0000000 --- a/src/cli/forecast.py +++ /dev/null @@ -1,280 +0,0 @@ -""" -Forecast Command - A derived view over the predictions ledger and question -graph. No separate forecast engine; what you see here is exactly what the -synthesis has already committed to as falsifiable observables, organized -by whether each one has resolved. -""" - -from datetime import datetime, timedelta - -import click - -from ..context.beat_scope import prediction_scope_filter, question_scope_filter -from ..database.connection import get_db -from ..database.models import ( - PREDICTION_STATUS_CONTRADICTED, - PREDICTION_STATUS_OPEN, - PREDICTION_STATUS_TRIGGERED, - QUESTION_STATUS_OPEN, - Prediction, - Question, -) -from ..utils import utcnow -from ..utils.cadence import InvalidCadence, describe_next_review, is_due -from .colors import accent, header, muted, success, warning -from .scope import beat_option, resolve_beat_scope, scope_label -from .stake import due_predictions_header, render_due_prediction - - -@click.command(name="forecast") -@click.option( - "--days", - "-d", - type=int, - default=60, - help="Window for the resolved track record (default: 60 days).", -) -@click.option( - "--due", - "due_only", - is_flag=True, - default=False, - help=( - "Show only what is due right now: your claims past their resolution " - "date, and questions whose own review interval has elapsed. Surfacing " - "a question stamps it as reviewed." - ), -) -@beat_option -def forecast_command(days, due_only, beat_name): - """ - Show open observables and the recently-resolved record. - - Scoped: this is a derived view over the predictions ledger, so it inherits - the ledger's scoping. Without --beat it shows your own observables only. - """ - if due_only: - _render_due(beat_name) - return - - cutoff = utcnow() - timedelta(days=days) - - with get_db() as session: - beat_id = resolve_beat_scope(session, beat_name) - scope = prediction_scope_filter(session, beat_id) - open_preds = ( - session.query(Prediction) - .filter(Prediction.status == PREDICTION_STATUS_OPEN, scope) - .order_by(Prediction.made_at.asc()) - .all() - ) - resolved = ( - session.query(Prediction) - .filter( - Prediction.status.in_( - [PREDICTION_STATUS_TRIGGERED, PREDICTION_STATUS_CONTRADICTED] - ), - Prediction.resolved_at >= cutoff, - scope, - ) - .order_by(Prediction.resolved_at.desc()) - .all() - ) - - click.echo(header(f"FORECAST ({scope_label(beat_name)})")) - click.echo( - muted( - "A derived view over the predictions ledger -- not a separate engine. " - "What you see here is exactly what past syntheses committed to as " - "falsifiable observables." - ) - ) - click.echo("=" * 80) - click.echo() - - click.echo(header("KNOWN UNKNOWNS")) - click.echo(muted("Open observables -- explicit triggers, not yet resolved.")) - click.echo() - if not open_preds: - click.echo(muted(" No open observables.")) - else: - for p in open_preds: - _render_open_prediction(p) - click.echo() - - click.echo(header(f"KNOWN KNOWNS (last {days} days)")) - click.echo(muted("Observables later coverage triggered or contradicted.")) - click.echo() - if not resolved: - click.echo(muted(" No resolutions in this window.")) - else: - triggered = [p for p in resolved if p.status == PREDICTION_STATUS_TRIGGERED] - contradicted = [p for p in resolved if p.status == PREDICTION_STATUS_CONTRADICTED] - for p in triggered: - _render_resolved_prediction(p, success("triggered")) - for p in contradicted: - _render_resolved_prediction(p, warning("contradicted")) - - click.echo() - click.echo( - muted( - "Note: the 'unknown unknowns' bucket has been removed by design. " - "The tool does not fabricate observables it cannot ground in evidence." - ) - ) - - -def _render_due(beat_name: str | None) -> None: - """ - Answer "what is due right now", at each question's own speed. - - Two things can be due, and they are deliberately different clocks: - - * a **prediction** whose ``due_by`` has passed -- a specific claim that can - now be graded, and - * a **question** whose own review interval has elapsed since it was last - reviewed -- worth re-examining, whether or not anything moved. - - Surfacing a question stamps ``last_reviewed_at``. That is the point: a - quiet question that reappears every day trains the operator to skim, and - skimming is how a standing agenda stops being one. Nothing runs on a - schedule -- ``--due`` is a question you ask. - """ - now = utcnow() - with get_db() as session: - beat_id = resolve_beat_scope(session, beat_name) - - awaiting = ( - session.query(Prediction) - .filter( - Prediction.status == PREDICTION_STATUS_OPEN, - Prediction.due_by.is_not(None), - Prediction.due_by <= now, - prediction_scope_filter(session, beat_id), - ) - .order_by(Prediction.due_by.asc()) - .all() - ) - - open_questions = ( - session.query(Question) - .filter( - Question.status == QUESTION_STATUS_OPEN, - question_scope_filter(session, beat_id), - ) - .order_by(Question.first_asked_at.asc()) - .all() - ) - # Each question is tested against its *own* interval, not a global - # rhythm: two questions on different cadences come due independently. - due_questions = [q for q in open_questions if _question_is_due(q, now)] - uncadenced = sum(1 for q in open_questions if not q.cadence) - - click.echo(header(f"DUE NOW ({scope_label(beat_name)})")) - click.echo(muted(f"As of {now.date().isoformat()}. Nothing here runs on its own.")) - click.echo("=" * 80) - click.echo() - - due_predictions_header(len(awaiting)) - if not awaiting: - click.echo(muted(" Nothing of yours has come due.")) - click.echo() - else: - for p in awaiting: - render_due_prediction(p, now) - - click.echo(header(f"QUESTIONS DUE FOR REVIEW ({len(due_questions)})")) - click.echo(muted("Each question against its own cadence, not a shared rhythm.")) - click.echo() - if not due_questions: - click.echo(muted(" No question's review interval has elapsed.")) - click.echo() - else: - for q in due_questions: - _render_due_question(session, q, now, beat_id) - - # The stamp happens whether or not anything moved. - for q in due_questions: - q.last_reviewed_at = now - session.commit() - - if due_questions: - ids = ", ".join(f"Q{q.id}" for q in due_questions) - click.echo(success(f"Stamped as reviewed: {ids}.")) - click.echo( - muted("Reviewed means looked at, not resolved. They return at their next interval.") - ) - click.echo() - - if uncadenced: - click.echo( - muted( - f"{uncadenced} open question(s) carry no cadence and never surface " - "here. A cadence is your read on how fast a subject moves, so only " - "questions you declared with 'questions add' have one." - ) - ) - - -def _question_is_due(q: Question, now: datetime) -> bool: - """Whether one question's own interval has elapsed. A bad stored cadence is not due.""" - try: - return is_due(q.cadence, q.last_reviewed_at, q.first_asked_at, now) - except InvalidCadence: - return False - - -def _render_due_question(session, q: Question, now: datetime, beat_id: int | None) -> None: - """A due question plus whatever open claims are already staked on it.""" - try: - phrase = describe_next_review(q.cadence, q.last_reviewed_at, q.first_asked_at, now) - except InvalidCadence as exc: - phrase = f"unreadable cadence ({exc.raw})" - last = q.last_reviewed_at.date().isoformat() if q.last_reviewed_at else "never" - click.echo( - f" {accent(f'Q{q.id}')} {muted(f'cadence {q.cadence} | {phrase} | last reviewed {last}')}" - ) - click.echo(f" {q.text}") - - open_claims = ( - session.query(Prediction) - .filter( - Prediction.question_id == q.id, - Prediction.status == PREDICTION_STATUS_OPEN, - prediction_scope_filter(session, beat_id), - ) - .order_by(Prediction.made_at.asc()) - .all() - ) - if not open_claims: - click.echo( - muted(f' No open claim. Stake one: predict {q.id} "..." --by DATE --confidence N') - ) - for p in open_claims: - due = p.due_by.date().isoformat() if p.due_by else "no date" - conf = f"{p.confidence:.0%}" if p.confidence is not None else "no confidence" - click.echo(muted(f" P{p.id} [{p.author}] {conf}, due {due}: {p.observable_text}")) - click.echo() - - -def _question_tag(question: Question | None) -> str: - if question is None: - return "" - return f"[Q{question.id}] " - - -def _render_open_prediction(p: Prediction) -> None: - tag = _question_tag(p.question) - click.echo(f" {accent(f'P{p.id}')} {muted(tag)}{p.observable_text}") - click.echo(f" Trigger: {muted(p.trigger_condition)}") - if p.question: - click.echo(f" Bears on: {muted(p.question.text)}") - click.echo() - - -def _render_resolved_prediction(p: Prediction, status_label: str) -> None: - tag = _question_tag(p.question) - click.echo(f" {accent(f'P{p.id}')} [{status_label}] {muted(tag)}{p.observable_text}") - if p.resolution_note: - click.echo(f" {p.resolution_note}") - click.echo() diff --git a/src/cli/frames.py b/src/cli/frames.py deleted file mode 100644 index 5af6e7d..0000000 --- a/src/cli/frames.py +++ /dev/null @@ -1,409 +0,0 @@ -""" -Frames Command - Narrative Frame Glossary Management -Browse, validate, and edit the emergent frame glossary. -""" - -import os -import subprocess -import tempfile - -import click - -from ..database.connection import get_db -from ..database.models import ArticleFrame, FrameGap, NarrativeFrame, TopicCluster -from .colors import accent, error, header, muted, success, warning - - -@click.group(name="frames") -def frames_command(): - """Manage the narrative frame glossary.""" - pass - - -@frames_command.command(name="list") -@click.option("--unvalidated", "-u", is_flag=True, help="Show only unvalidated frames") -def list_frames(unvalidated): - """List all topic clusters and their frames.""" - with get_db() as session: - clusters = session.query(TopicCluster).order_by(TopicCluster.name).all() - - if not clusters: - click.echo(muted("No topic clusters found. Run a brief to discover frames.")) - return - - click.echo(header("NARRATIVE FRAME GLOSSARY")) - click.echo("=" * 70) - - for cluster in clusters: - frames = ( - session.query(NarrativeFrame) - .filter_by(topic_cluster_id=cluster.id) - .order_by(NarrativeFrame.label) - .all() - ) - - if unvalidated: - frames = [f for f in frames if not f.validated] - if not frames: - continue - - validated_count = sum(1 for f in frames if f.validated) - total = len(frames) - - click.echo() - click.echo(f"{accent(cluster.name)} {muted(f'[{validated_count}/{total} validated]')}") - - if not frames: - click.echo(muted(" (no frames discovered yet)")) - continue - - for frame in frames: - status = success("[v]") if frame.validated else warning("[?]") - click.echo(f" {status} #{frame.id} {frame.label}") - - click.echo() - click.echo(muted("Use 'frames show ' for details or 'frames edit ' to modify.")) - - -@frames_command.command(name="show") -@click.argument("topic") -def show_topic(topic): - """Show frames for a topic cluster. - - TOPIC can be a cluster name (partial match) or numeric ID. - """ - with get_db() as session: - cluster = _find_cluster(session, topic) - if not cluster: - click.echo(error(f"No topic cluster matching '{topic}'")) - return - - frames = ( - session.query(NarrativeFrame) - .filter_by(topic_cluster_id=cluster.id) - .order_by(NarrativeFrame.first_seen) - .all() - ) - - click.echo(header(f"TOPIC: {cluster.name}")) - click.echo("=" * 70) - click.echo(f"Keywords: {', '.join(cluster.keywords or [])}") - click.echo(f"Created: {cluster.created_at.strftime('%Y-%m-%d')}") - click.echo() - - if not frames: - click.echo(muted("No frames discovered for this topic yet.")) - return - - for frame in frames: - status = success("VALIDATED") if frame.validated else warning("UNVALIDATED") - click.echo(f"{accent(f'#{frame.id}')} {frame.label} [{status}]") - click.echo(f" {frame.description or '(no description)'}") - if frame.assumptions: - click.echo(f" Assumes: {muted(frame.assumptions)}") - click.echo(f" First seen: {frame.first_seen.strftime('%Y-%m-%d')}") - - # Show article count using this frame - article_count = session.query(ArticleFrame).filter_by(frame_id=frame.id).count() - if article_count: - click.echo(f" Articles tagged: {article_count}") - click.echo() - - # Show gaps for this cluster - gaps = ( - session.query(FrameGap) - .filter_by(topic_cluster_id=cluster.id) - .order_by(FrameGap.occurrences.desc()) - .all() - ) - - if gaps: - click.echo(header("FRAME GAPS")) - click.echo("-" * 70) - for gap in gaps: - click.echo( - f" {warning('GAP')} {gap.frame_label} " - f"({gap.occurrences} occurrence{'s' if gap.occurrences != 1 else ''})" - ) - if gap.feed_suggestion: - click.echo(f" Feed suggestion: {muted(gap.feed_suggestion)}") - click.echo() - - -@frames_command.command(name="edit") -@click.argument("frame_id", type=int) -def edit_frame(frame_id): - """Edit a narrative frame in $EDITOR. - - Opens the frame's label, description, and assumptions in your editor. - Same pattern as git commit. - """ - with get_db() as session: - frame = session.query(NarrativeFrame).filter_by(id=frame_id).first() - if not frame: - click.echo(error(f"No frame with ID {frame_id}")) - return - - cluster = session.query(TopicCluster).filter_by(id=frame.topic_cluster_id).first() - topic_name = cluster.name if cluster else "unknown" - - # Build editable content - content = ( - f"# Editing frame #{frame.id} for topic: {topic_name}\n" - f"# Lines starting with # are comments and will be ignored.\n" - f"# Save and close your editor to apply changes.\n" - f"# Leave the file empty to cancel.\n" - f"\n" - f"label: {frame.label}\n" - f"description: {frame.description or ''}\n" - f"assumptions: {frame.assumptions or ''}\n" - ) - - editor = os.environ.get("EDITOR", os.environ.get("VISUAL", "vi")) - - with tempfile.NamedTemporaryFile( - mode="w", suffix=".txt", prefix="insightweaver-frame-", delete=False - ) as tmp: - tmp.write(content) - tmp_path = tmp.name - - try: - result = subprocess.run([editor, tmp_path]) - if result.returncode != 0: - click.echo(error("Editor exited with an error. No changes applied.")) - return - - with open(tmp_path) as f: - edited = f.read() - - # Parse edited content - parsed = _parse_frame_edit(edited) - if not parsed: - click.echo(muted("Empty or unchanged. No changes applied.")) - return - - # Apply changes - frame.label = parsed.get("label", frame.label) - frame.description = parsed.get("description", frame.description) - frame.assumptions = parsed.get("assumptions", frame.assumptions) - session.commit() - - click.echo(success(f"Frame #{frame_id} updated.")) - click.echo(f" Label: {frame.label}") - click.echo(f" Description: {frame.description}") - click.echo(f" Assumptions: {frame.assumptions}") - - finally: - os.unlink(tmp_path) - - -@frames_command.command(name="gaps") -@click.option("--min-occurrences", "-m", type=int, default=1, help="Minimum gap occurrences") -def show_gaps(min_occurrences): - """Show recurring frame gaps across all topics. - - Gaps are perspectives consistently absent from your feeds. - High-occurrence gaps suggest you should add a feed source. - """ - with get_db() as session: - gaps = ( - session.query(FrameGap, TopicCluster) - .join(TopicCluster, FrameGap.topic_cluster_id == TopicCluster.id) - .filter(FrameGap.occurrences >= min_occurrences) - .order_by(FrameGap.occurrences.desc()) - .all() - ) - - if not gaps: - click.echo(muted("No frame gaps detected yet. Run briefs to discover gaps.")) - return - - click.echo(header("FRAME GAPS -- FEED CURATION SIGNALS")) - click.echo("=" * 70) - click.echo() - - for gap, cluster in gaps: - click.echo( - f" {warning(f'[{gap.occurrences}x]')} " - f"{gap.frame_label} " - f"{muted(f'({cluster.name})')}" - ) - if gap.feed_suggestion: - click.echo(f" Suggested feed type: {accent(gap.feed_suggestion)}") - click.echo(f" First detected: {gap.first_detected.strftime('%Y-%m-%d')}") - click.echo() - - click.echo( - muted( - "These gaps represent perspectives absent from your current feeds.\n" - "Consider adding RSS sources for the suggested feed types." - ) - ) - - -def _find_cluster(session, identifier: str) -> TopicCluster | None: - """Find a topic cluster by ID or partial name match.""" - # Try numeric ID first - try: - cluster_id = int(identifier) - return session.query(TopicCluster).filter_by(id=cluster_id).first() - except ValueError: - pass - - # Try exact name match - cluster = session.query(TopicCluster).filter(TopicCluster.name.ilike(identifier)).first() - if cluster: - return cluster - - # Try partial match - return session.query(TopicCluster).filter(TopicCluster.name.ilike(f"%{identifier}%")).first() - - -def _parse_frame_edit(content: str) -> dict | None: - """Parse the edited frame file content. - - Returns dict with label/description/assumptions, or None if empty. - """ - lines = [ - line for line in content.strip().splitlines() if not line.startswith("#") and line.strip() - ] - - if not lines: - return None - - result = {} - for line in lines: - if ":" in line: - key, _, value = line.partition(":") - key = key.strip().lower() - value = value.strip() - if key in ("label", "description", "assumptions") and value: - result[key] = value - - return result if result else None - - -def run_frame_validation_loop(): - """ - Interactive frame validation loop. - - Called at the end of a brief run when unvalidated frames exist. - Shows each candidate frame and prompts: y/n/edit/skip. - 'edit' opens $EDITOR with the frame pre-populated (same pattern as git commit). - 'skip' exits the loop, leaving remaining frames for next time. - - Returns the number of frames validated. - """ - try: - return _run_frame_validation_loop_inner() - except Exception: - # Frame tables may not exist yet -- don't crash the brief - return 0 - - -def _run_frame_validation_loop_inner(): - with get_db() as session: - unvalidated = ( - session.query(NarrativeFrame, TopicCluster) - .join(TopicCluster, NarrativeFrame.topic_cluster_id == TopicCluster.id) - .filter(NarrativeFrame.validated.is_(False)) - .order_by(NarrativeFrame.first_seen.desc()) - .all() - ) - - if not unvalidated: - return 0 - - click.echo() - click.echo(header("FRAME VALIDATION")) - click.echo("=" * 70) - click.echo( - f"{len(unvalidated)} unvalidated frame{'s' if len(unvalidated) != 1 else ''} " - f"discovered. Review each one:" - ) - click.echo(muted(" y = accept | n = reject | edit = open in $EDITOR | skip = stop")) - click.echo() - - validated_count = 0 - - for frame, cluster in unvalidated: - click.echo(f"{accent(f'#{frame.id}')} {frame.label}") - click.echo(f" Topic: {cluster.name}") - click.echo(f" Description: {frame.description or '(none)'}") - click.echo(f" Assumes: {muted(frame.assumptions or '(none)')}") - - # Show article count - article_count = session.query(ArticleFrame).filter_by(frame_id=frame.id).count() - if article_count: - click.echo(f" Articles tagged: {article_count}") - - click.echo() - - while True: - choice = click.prompt( - " Action", type=click.Choice(["y", "n", "edit", "skip"]), show_choices=True - ) - - if choice == "y": - frame.validated = True - session.commit() - click.echo(success(" Accepted.")) - validated_count += 1 - break - elif choice == "n": - session.delete(frame) - session.commit() - click.echo(muted(" Rejected and removed.")) - break - elif choice == "edit": - _edit_frame_inline(session, frame, cluster.name) - # After edit, mark as validated - frame.validated = True - session.commit() - click.echo(success(" Edited and accepted.")) - validated_count += 1 - break - elif choice == "skip": - click.echo(muted(" Skipping remaining frames.")) - return validated_count - - click.echo() - - if validated_count: - click.echo(success(f"Validated {validated_count} frame(s).")) - - return validated_count - - -def _edit_frame_inline(_session, frame: NarrativeFrame, topic_name: str): - """Open a frame in $EDITOR for editing.""" - content = ( - f"# Editing frame #{frame.id} for topic: {topic_name}\n" - f"# Lines starting with # are comments and will be ignored.\n" - f"\n" - f"label: {frame.label}\n" - f"description: {frame.description or ''}\n" - f"assumptions: {frame.assumptions or ''}\n" - ) - - editor = os.environ.get("EDITOR", os.environ.get("VISUAL", "vi")) - - with tempfile.NamedTemporaryFile( - mode="w", suffix=".txt", prefix="insightweaver-frame-", delete=False - ) as tmp: - tmp.write(content) - tmp_path = tmp.name - - try: - subprocess.run([editor, tmp_path]) - - with open(tmp_path) as f: - edited = f.read() - - parsed = _parse_frame_edit(edited) - if parsed: - frame.label = parsed.get("label", frame.label) - frame.description = parsed.get("description", frame.description) - frame.assumptions = parsed.get("assumptions", frame.assumptions) - finally: - os.unlink(tmp_path) diff --git a/src/cli/loading.py b/src/cli/loading.py deleted file mode 100644 index 1210db1..0000000 --- a/src/cli/loading.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Loading indicator utility for CLI commands -Provides animated "Executing..." display with dots -""" - -import sys -import threading -import time -from contextlib import contextmanager - -# ANSI escape codes for bright green (hacker aesthetic) -GREEN = "\033[92m" -RESET = "\033[0m" - - -class LoadingIndicator: - """Animated loading indicator with dots""" - - def __init__(self, message="Executing"): - self.message = message - self.is_running = False - self.thread = None - - def _animate(self): - """Animation loop - spinner cycles continuously""" - frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"] - idx = 0 - while self.is_running: - frame = frames[idx % len(frames)] - # Write to __stderr__ which is the original stderr before any redirection - # This prevents the output manager from suppressing the spinner - sys.__stderr__.write(f"\r{GREEN}{frame}{RESET} {self.message}... ") - sys.__stderr__.flush() - - idx += 1 - time.sleep(0.1) - - # Clear the line when done - sys.__stderr__.write("\r" + " " * (len(self.message) + 10) + "\r") - sys.__stderr__.flush() - - def start(self): - """Start the loading animation""" - self.is_running = True - self.thread = threading.Thread(target=self._animate, daemon=True) - self.thread.start() - - def stop(self): - """Stop the loading animation""" - self.is_running = False - if self.thread: - self.thread.join(timeout=1) - - -@contextmanager -def loading(message="Executing", debug=False): - """ - Context manager for showing loading indicator - - Args: - message: Message to display (default: "Executing") - debug: If True, skip loading indicator - - Usage: - with loading("Processing data"): - # do work - pass - """ - if debug: - yield - return - - indicator = LoadingIndicator(message) - indicator.start() - try: - yield - finally: - indicator.stop() diff --git a/src/cli/predictions.py b/src/cli/predictions.py deleted file mode 100644 index b80ff95..0000000 --- a/src/cli/predictions.py +++ /dev/null @@ -1,276 +0,0 @@ -""" -Predictions Command - Inspect the predictions ledger and the tool's own -calibration track record. -""" - -from datetime import timedelta - -import click - -from ..context.beat_scope import prediction_scope_filter -from ..database.connection import get_db -from ..database.models import ( - PREDICTION_AUTHOR_MODEL, - PREDICTION_AUTHOR_OPERATOR, - PREDICTION_OUTCOME_NO, - PREDICTION_OUTCOME_YES, - PREDICTION_STATUS_CONTRADICTED, - PREDICTION_STATUS_EXPIRED, - PREDICTION_STATUS_OPEN, - PREDICTION_STATUS_TRIGGERED, - Prediction, -) -from ..utils import utcnow -from .colors import accent, header, muted, success, warning -from .scope import beat_option, resolve_beat_scope, scope_label - - -@click.group(name="predictions") -def predictions_command(): - """Inspect the predictions ledger and calibration record.""" - pass - - -def _print_predictions(rows, show_resolution: bool): - now = utcnow() - for p in rows: - age_days = (now - p.made_at).days - question_text = p.question.text if p.question else "(unknown question)" - click.echo( - f"{accent(f'P{p.id}')} [{p.author}] " - f"{muted(f'made {p.made_at.date().isoformat()} ({age_days}d ago)')}" - ) - click.echo(f" Observable: {p.observable_text}") - click.echo(f" Trigger: {muted(p.trigger_condition)}") - if p.due_by is not None or p.confidence is not None: - due = p.due_by.date().isoformat() if p.due_by else "no date" - conf = f"{p.confidence:.0%}" if p.confidence is not None else "no confidence" - click.echo(muted(f" Staked at {conf}, resolving {due}")) - click.echo(f" Bears on: {muted(question_text)}") - if show_resolution and p.outcome: - click.echo(muted(f" Outcome: {p.outcome} (recorded {_resolved_stamp(p)})")) - if show_resolution and p.resolution_note: - click.echo(f" {p.resolution_note}") - click.echo() - - -def _resolved_stamp(p: Prediction) -> str: - """Resolution date, and how it sat against the due date.""" - if p.resolved_at is None: - return "date unknown" - stamp = p.resolved_at.date().isoformat() - if p.due_by is None: - return stamp - delta = (p.resolved_at.date() - p.due_by.date()).days - if delta > 0: - return f"{stamp}, {delta}d after it came due" - if delta < 0: - return f"{stamp}, {-delta}d before it came due" - return f"{stamp}, on the day" - - -@predictions_command.command(name="open") -@click.option("--limit", "-n", type=int, default=50) -@beat_option -def open_predictions(limit, beat_name): - """ - List predictions still waiting on coverage. - - Scoped: without --beat this is your own ledger only. - """ - with get_db() as session: - beat_id = resolve_beat_scope(session, beat_name) - rows = ( - session.query(Prediction) - .filter( - Prediction.status == PREDICTION_STATUS_OPEN, - prediction_scope_filter(session, beat_id), - ) - .order_by(Prediction.made_at.asc()) - .limit(limit) - .all() - ) - if not rows: - click.echo(muted("No open predictions in " + scope_label(beat_name) + ".")) - return - click.echo(header(f"OPEN PREDICTIONS ({len(rows)} shown, {scope_label(beat_name)})")) - click.echo("=" * 80) - _print_predictions(rows, show_resolution=False) - - -@predictions_command.command(name="triggered") -@click.option("--limit", "-n", type=int, default=50) -@beat_option -def triggered_predictions(limit, beat_name): - """ - List predictions whose observable showed up in later coverage. - - Scoped: without --beat this is your own ledger only. - """ - with get_db() as session: - beat_id = resolve_beat_scope(session, beat_name) - rows = ( - session.query(Prediction) - .filter( - Prediction.status == PREDICTION_STATUS_TRIGGERED, - prediction_scope_filter(session, beat_id), - ) - .order_by(Prediction.resolved_at.desc()) - .limit(limit) - .all() - ) - if not rows: - click.echo(muted("No triggered predictions yet in " + scope_label(beat_name) + ".")) - return - click.echo(header(f"TRIGGERED PREDICTIONS ({len(rows)} shown, {scope_label(beat_name)})")) - click.echo("=" * 80) - _print_predictions(rows, show_resolution=True) - - -@predictions_command.command(name="contradicted") -@click.option("--limit", "-n", type=int, default=50) -@beat_option -def contradicted_predictions(limit, beat_name): - """ - List predictions later coverage explicitly went against. - - Scoped: without --beat this is your own ledger only. - """ - with get_db() as session: - beat_id = resolve_beat_scope(session, beat_name) - rows = ( - session.query(Prediction) - .filter( - Prediction.status == PREDICTION_STATUS_CONTRADICTED, - prediction_scope_filter(session, beat_id), - ) - .order_by(Prediction.resolved_at.desc()) - .limit(limit) - .all() - ) - if not rows: - click.echo(muted("No contradicted predictions yet in " + scope_label(beat_name) + ".")) - return - click.echo( - header(f"CONTRADICTED PREDICTIONS ({len(rows)} shown, {scope_label(beat_name)})") - ) - click.echo("=" * 80) - _print_predictions(rows, show_resolution=True) - - -@predictions_command.command(name="track-record") -@click.option("--days", "-d", type=int, default=90, help="Window in days (default: 90).") -@beat_option -def track_record(days, beat_name): - """ - Show the tool's calibration record over a rolling window. - - Scoped, and this is the case that matters most: a calibration number is - only meaningful for one ledger. Mixing a beat's resolved observables into - your own hit rate would corrupt the one figure the tool exists to be - honest about. - - Split by author since 2026-08-27 (backlog task 011): the calibration figure - counts operator predictions only. Model predictions are reported under - their own heading and are never mixed in -- a track record blending the - two measures nothing. - """ - cutoff = utcnow() - timedelta(days=days) - with get_db() as session: - beat_id = resolve_beat_scope(session, beat_name) - window = session.query(Prediction).filter( - Prediction.made_at >= cutoff, - prediction_scope_filter(session, beat_id), - ) - if window.count() == 0: - click.echo( - muted(f"No predictions made in the last {days} days in {scope_label(beat_name)}.") - ) - return - - click.echo(header(f"CALIBRATION RECORD (last {days} days, {scope_label(beat_name)})")) - click.echo("=" * 80) - click.echo() - _render_operator_record(window) - click.echo() - _render_model_record(window) - - -def _status_counts(window) -> dict[str, int]: - """Status tally for one slice of the ledger.""" - return { - "open": window.filter(Prediction.status == PREDICTION_STATUS_OPEN).count(), - "triggered": window.filter(Prediction.status == PREDICTION_STATUS_TRIGGERED).count(), - "contradicted": window.filter(Prediction.status == PREDICTION_STATUS_CONTRADICTED).count(), - "expired": window.filter(Prediction.status == PREDICTION_STATUS_EXPIRED).count(), - } - - -def _render_operator_record(window) -> None: - """ - The calibration figure. Operator predictions only. - - A plain hit rate on purpose -- how often the claim, as written, came true. - No weighting or Brier-style aggregation; that is a later decision once - there is real data to aggregate. - """ - mine = window.filter(Prediction.author == PREDICTION_AUTHOR_OPERATOR) - total = mine.count() - - click.echo(header("YOUR CALIBRATION (operator predictions only)")) - if total == 0: - click.echo( - muted( - " You have staked nothing in this window. The calibration figure " - 'counts only claims you made: \'predict "..." ' - "--by DATE --confidence N'." - ) - ) - return - - right = mine.filter(Prediction.outcome == PREDICTION_OUTCOME_YES).count() - wrong = mine.filter(Prediction.outcome == PREDICTION_OUTCOME_NO).count() - counts = _status_counts(mine) - resolved = right + wrong - - click.echo(f" Claims staked: {total}") - click.echo(f" {success('Came true')}: {right}") - click.echo(f" {warning('Did not')}: {wrong}") - click.echo(f" {muted('Still open')}: {counts['open']}") - if counts["expired"]: - click.echo(f" {muted('Expired unjudged')}: {counts['expired']}") - click.echo() - if resolved: - click.echo(f" Hit rate: {right}/{resolved} = {right / resolved:.0%}.") - else: - click.echo(muted(" Nothing of yours has been resolved yet in this window.")) - - -def _render_model_record(window) -> None: - """ - The model's predictions, reported separately and never folded into the - figure above. They stay in the ledger as prompts -- suggestions about what - is worth holding an opinion on -- not as claims anyone staked. - """ - theirs = window.filter(Prediction.author == PREDICTION_AUTHOR_MODEL) - total = theirs.count() - - click.echo(header("MODEL PREDICTIONS (not counted toward your calibration)")) - if total == 0: - click.echo(muted(" The model made none in this window.")) - return - - counts = _status_counts(theirs) - click.echo( - muted( - f" {total} made | {counts['triggered']} triggered | " - f"{counts['contradicted']} contradicted | {counts['open']} open | " - f"{counts['expired']} expired" - ) - ) - click.echo( - muted( - " Reported for transparency only. These are the tool's own " - "observables, not your judgement." - ) - ) diff --git a/src/cli/questions.py b/src/cli/questions.py deleted file mode 100644 index 107fb18..0000000 --- a/src/cli/questions.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -Questions Command - Inspect and manage the persistent Question graph. -""" - -import click - -from ..context.beat_scope import owning_beat_names, question_scope_filter -from ..context.question_matcher import normalize_question -from ..database.connection import get_db -from ..database.models import ( - QUESTION_STATUS_OPEN, - QUESTION_STATUS_RESOLVED, - NarrativeSynthesis, - Question, - QuestionSituation, -) -from ..utils import utcnow -from ..utils.cadence import ( - CADENCE_EXAMPLES, - InvalidCadence, - describe_next_review, - normalize_cadence, -) -from .colors import accent, header, muted, success, warning -from .scope import beat_option, resolve_beat_scope, scope_label - - -@click.group(name="questions") -def questions_command(): - """Inspect and manage the persistent question graph.""" - pass - - -@questions_command.command(name="list") -@click.option("--status", "-s", type=click.Choice(["open", "resolved", "all"]), default="open") -@click.option("--limit", "-n", type=int, default=50) -@beat_option -def list_questions(status, limit, beat_name): - """ - List questions, oldest open first by default. - - Scoped: without --beat this is your own ledger and never shows a beat's - questions. Pass --beat NAME to read that subject's ledger instead. - """ - with get_db() as session: - beat_id = resolve_beat_scope(session, beat_name) - query = session.query(Question).filter(question_scope_filter(session, beat_id)) - if status != "all": - query = query.filter(Question.status == status) - - if status == QUESTION_STATUS_OPEN: - query = query.order_by(Question.first_asked_at.asc()) - else: - query = query.order_by(Question.first_asked_at.desc()) - - rows = query.limit(limit).all() - - if not rows: - click.echo(muted(f"No {status} questions in {scope_label(beat_name)}.")) - return - - click.echo(header(f"QUESTIONS ({status}, {len(rows)} shown, {scope_label(beat_name)})")) - click.echo("=" * 80) - now = utcnow() - for q in rows: - age_days = (now - q.first_asked_at).days - tag = success("open") if q.status == QUESTION_STATUS_OPEN else muted(q.status) - click.echo( - f"{accent(f'Q{q.id}')} [{tag}] " - f"{muted(f'asked {q.first_asked_at.date().isoformat()} ({age_days}d ago)')}" - ) - click.echo(f" {q.text}") - prefix = f"cadence {q.cadence} | " if q.cadence else "" - click.echo(muted(f" {prefix}{_review_phrase(q, now)}")) - if q.previous_question_id: - click.echo(muted(f" (follows Q{q.previous_question_id})")) - click.echo() - - -def _review_phrase(q: Question, now) -> str: - """Time-until-next-review for one question, tolerant of a bad stored cadence.""" - try: - return describe_next_review(q.cadence, q.last_reviewed_at, q.first_asked_at, now) - except InvalidCadence as exc: - return f"unreadable cadence ({exc.raw})" - - -@questions_command.command(name="add") -@click.argument("text") -@click.option( - "--cadence", - "cadence", - required=True, - metavar="INTERVAL", - help=( - "How often this question is worth re-examining, e.g. " - f"{', '.join(CADENCE_EXAMPLES)}. Required: the interval is your read on " - "how fast the subject moves, so the tool will not guess one." - ), -) -def add_question(text, cadence): - """ - Declare a question you want carried, at your own review cadence. - - Lands in your own ledger. A beat's agenda is declared in the beat config, - not here. - """ - claim = (text or "").strip() - if not claim: - raise click.UsageError("A question needs text. Nothing was stored.") - - try: - interval = normalize_cadence(cadence) - except InvalidCadence as exc: - raise click.UsageError(f"{exc} Nothing was stored.") - - with get_db() as session: - q = Question( - text=claim, - normalized_text=normalize_question(claim), - first_asked_at=utcnow(), - status=QUESTION_STATUS_OPEN, - cadence=interval, - # Never reviewed: the first interval counts from now, so a question - # declared today at 90d is not due today. - last_reviewed_at=None, - ) - session.add(q) - session.commit() - click.echo(success(f"Added question Q{q.id} at a {interval} review cadence.")) - click.echo(muted(f" {claim}")) - click.echo( - muted(f' Stake a claim on it with: predict {q.id} "..." --by DATE --confidence N') - ) - - -@questions_command.command(name="show") -@click.argument("question_id", type=int) -def show_question(question_id): - """ - Show a question's full history across runs. - - Not scoped: you named a specific question, so it is looked up wherever it - lives. The ledger it belongs to is disclosed rather than assumed. - """ - with get_db() as session: - q = session.query(Question).filter_by(id=question_id).first() - if not q: - click.echo(warning(f"Question Q{question_id} not found.")) - return - - click.echo(header(f"QUESTION Q{q.id}")) - click.echo("=" * 80) - click.echo(q.text) - click.echo() - owners = owning_beat_names(session, q.id) - click.echo(muted(f"Ledger: {', '.join(owners) if owners else 'yours (no beat)'}")) - click.echo(muted(f"First asked: {q.first_asked_at.isoformat()}")) - click.echo(muted(f"Status: {q.status}")) - click.echo( - muted( - f"Cadence: {q.cadence or 'none'} | " - f"{_review_phrase(q, utcnow())} | " - f"last reviewed: " - f"{q.last_reviewed_at.date().isoformat() if q.last_reviewed_at else 'never'}" - ) - ) - if q.resolved_at: - click.echo(muted(f"Resolved: {q.resolved_at.isoformat()}")) - if q.resolution_note: - click.echo(muted(f"Note: {q.resolution_note}")) - if q.previous_question_id: - click.echo(muted(f"Previous question: Q{q.previous_question_id}")) - click.echo() - - links = ( - session.query(QuestionSituation, NarrativeSynthesis) - .join(NarrativeSynthesis, QuestionSituation.synthesis_id == NarrativeSynthesis.id) - .filter(QuestionSituation.question_id == q.id) - .order_by(QuestionSituation.observed_at.asc()) - .all() - ) - - if not links: - click.echo(muted("No appearances recorded.")) - return - - click.echo(header(f"Appearances ({len(links)})")) - click.echo("-" * 80) - for i, (link, synth) in enumerate(links, 1): - situation_title = _situation_title(synth, link.situation_index) - click.echo( - f" {accent(f'#{i}')} {link.observed_at.date().isoformat()} " - f"synthesis {synth.id} situation {link.situation_index}" - ) - if situation_title: - click.echo(f" {muted(situation_title)}") - - -@questions_command.command(name="resolve") -@click.argument("question_id", type=int) -@click.option("--note", required=True, help="Resolution note explaining what closed this question.") -def resolve_question(question_id, note): - """ - Mark a question resolved with a note. - - Not scoped, for the same reason as `show`: resolving is an explicit act on - a named row. Which ledger it belonged to is reported back, so resolving a - beat's question from your own context is visible rather than silent. - """ - with get_db() as session: - q = session.query(Question).filter_by(id=question_id).first() - if not q: - click.echo(warning(f"Question Q{question_id} not found.")) - return - if q.status != QUESTION_STATUS_OPEN: - click.echo(warning(f"Q{q.id} is already {q.status}; not modifying.")) - return - - owners = owning_beat_names(session, q.id) - q.status = QUESTION_STATUS_RESOLVED - q.resolved_at = utcnow() - q.resolution_note = note - session.commit() - - where = f" in beat {', '.join(owners)}" if owners else "" - click.echo(success(f"Q{q.id} resolved{where}.")) - - -def _situation_title(synth: NarrativeSynthesis, situation_index: int) -> str: - """Pull the title of a specific situation out of a stored synthesis blob.""" - data = synth.synthesis_data or {} - situations = data.get("situations", []) if isinstance(data, dict) else [] - if 0 <= situation_index < len(situations): - return situations[situation_index].get("title", "") - return "" diff --git a/src/cli/scope.py b/src/cli/scope.py deleted file mode 100644 index c74ef5e..0000000 --- a/src/cli/scope.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Shared ``--beat`` handling for the ledger-reading commands. - -``questions``, ``predictions`` and ``forecast`` read the commitment graph back -out. Once any beat has run, the graph holds more than one ledger, so those -commands have to say which one they mean. They answer the same way ``brief`` -does: no ``--beat`` is your own ledger, ``--beat NAME`` is that subject's. - -Added 2026-08-26 for backlog task 004 (read-side scoping). -""" - -import click - -from ..context.beat_scope import BeatNotRecorded, BeatTablesMissing, beat_id_by_name - -# One decorator, so the flag reads and behaves identically on every command -# that carries it. -beat_option = click.option( - "--beat", - "beat_name", - default=None, - metavar="NAME", - help="Read this beat's ledger instead of your own.", -) - - -def resolve_beat_scope(session, beat_name: str | None) -> int | None: - """ - Turn a ``--beat NAME`` value into a beat id, or None for the default scope. - - An unrecognised name is an error listing the beats that have actually run, - rather than an empty result set that would read as "you have nothing here". - """ - if beat_name is None: - return None - try: - return beat_id_by_name(session, beat_name) - except (BeatTablesMissing, BeatNotRecorded) as exc: - raise click.ClickException(str(exc)) - - -def scope_label(beat_name: str | None) -> str: - """How a heading names the ledger being shown.""" - return f"beat '{beat_name}'" if beat_name else "your ledger" diff --git a/src/cli/sources.py b/src/cli/sources.py index ec2e025..853a8a8 100644 --- a/src/cli/sources.py +++ b/src/cli/sources.py @@ -1,161 +1,126 @@ """ -Sources Command - Calibration by structural behavior, not by static dial. - -Two derived signals per feed, computed on demand from the article_frames -mapping and recorded frame gaps: - -- frame_uniqueness: fraction of this feed's frame-tagged articles whose - frame is carried by no other feed (a high score means this source is - a single point of exposure for those frames). -- gap_filling: fraction of recorded frame-gap labels that this feed's - articles have been tagged with (a high score means this source carries - perspectives the rest of the corpus has been missing). - -Claim survival -- the third signal from the original plan -- is deliberately -omitted. Computing it would require structured claim extraction we have not -built. We do not fabricate metrics; revisit if claim extraction lands. +Sources command - what is configured, and what it has actually stored. + +**This command was reduced, not designed, on 2026-08-31 (backlog task 012).** +It used to report two derived calibration signals per feed -- ``frame +uniqueness`` and ``gap filling`` -- both computed from the ``article_frames`` +mapping and recorded frame gaps. Task 012 deleted narrative frames, so both +signals lost their inputs. They are removed rather than reimplemented against +some other proxy: the original module refused to report claim survival because +computing it "would require structured claim extraction we have not built", and +inventing a replacement metric here would break the same rule in the same file. + +What is left is the part that does not depend on the deleted product: the feed +inventory and the corpus each feed has actually contributed. That is a real +answer to "which of my sources are pulling their weight" and it is measured, not +derived. Whatever the new pipeline wants to calibrate on, it can add here once +it exists. """ import click from sqlalchemy import func from ..database.connection import get_db -from ..database.models import ( - Article, - ArticleFrame, - FrameGap, - NarrativeFrame, - RSSFeed, -) +from ..database.models import Article, RSSFeed from .colors import accent, header, muted -def _compute_calibration(session) -> dict[int, dict]: - """Returns ``{feed_id: {name, uniqueness, gap_filling, tagged_articles}}``.""" +def _feed_stats(session) -> dict[int, dict]: + """Returns ``{feed_id: {name, category, is_active, articles, latest, ...}}``.""" feeds = session.query(RSSFeed).all() if not feeds: return {} - # frame_id -> distinct feed count - frame_feed_counts = dict( - session.query( - NarrativeFrame.id, - func.count(func.distinct(RSSFeed.id)), - ) - .join(ArticleFrame, ArticleFrame.frame_id == NarrativeFrame.id) - .join(Article, ArticleFrame.article_id == Article.id) - .join(RSSFeed, Article.feed_id == RSSFeed.id) - .group_by(NarrativeFrame.id) + counts = dict( + session.query(Article.feed_id, func.count(Article.id)).group_by(Article.feed_id).all() + ) + latest = dict( + session.query(Article.feed_id, func.max(Article.published_date)) + .group_by(Article.feed_id) .all() ) - unique_frame_ids = {fid for fid, n in frame_feed_counts.items() if n == 1} - - # Distinct gap frame_labels (case-normalized) we have on record. - gap_labels = { - row[0].strip().lower() - for row in session.query(FrameGap.frame_label).distinct().all() - if row[0] - } - result: dict[int, dict] = {} - for feed in feeds: - # All frame-tagged articles in this feed. - tagged_q = ( - session.query(ArticleFrame, NarrativeFrame.label, NarrativeFrame.id) - .join(NarrativeFrame, ArticleFrame.frame_id == NarrativeFrame.id) - .join(Article, ArticleFrame.article_id == Article.id) - .filter(Article.feed_id == feed.id) - .all() - ) - tagged = len(tagged_q) - unique_tags = sum(1 for _af, _label, fid in tagged_q if fid in unique_frame_ids) - uniqueness = (unique_tags / tagged) if tagged else 0.0 - - # Distinct frame labels this feed carries (case-normalized). - feed_labels = {label.strip().lower() for _af, label, _fid in tagged_q if label} - gap_filling = len(gap_labels & feed_labels) / len(gap_labels) if gap_labels else 0.0 - - result[feed.id] = { + return { + feed.id: { "name": feed.name, - "uniqueness": uniqueness, - "gap_filling": gap_filling, - "tagged_articles": tagged, + "category": feed.category, + "is_active": feed.is_active, + "articles": counts.get(feed.id, 0), + "latest": latest.get(feed.id), + "last_fetched": feed.last_fetched, + "error_count": feed.error_count or 0, + "last_error": feed.last_error, } - return result + for feed in feeds + } @click.group(name="sources") def sources_command(): - """Calibration of each feed by structural behavior.""" + """Configured feeds and the corpus they have contributed.""" pass @sources_command.command(name="list") def list_sources(): - """Show all feeds with derived calibration signals.""" + """Show all feeds with stored-article counts.""" with get_db() as session: - cal = _compute_calibration(session) - if not cal: + stats = _feed_stats(session) + if not stats: click.echo(muted("No feeds configured.")) return - click.echo(header("SOURCE CALIBRATION")) + click.echo(header("SOURCES")) click.echo( muted( - "Frame uniqueness: how much of what this source carries no other " - "feed does.\nGap filling: share of recorded frame gaps this source covers." + "Stored articles per feed. A feed with zero has been configured " + "but has contributed nothing to the corpus." ) ) click.echo("=" * 80) - # Sort by uniqueness descending so the standout sources surface first. - for _fid, row in sorted(cal.items(), key=lambda kv: -kv[1]["uniqueness"]): - tagged = row["tagged_articles"] - click.echo(accent(row["name"])) - if tagged == 0: - click.echo(muted(" no frame-tagged articles yet")) + # Most productive first, so the silent feeds collect at the bottom where + # they read as the finding they are. + for _fid, row in sorted(stats.items(), key=lambda kv: -kv[1]["articles"]): + label = row["name"] if row["is_active"] else f"{row['name']} (inactive)" + click.echo(accent(label)) + if row["articles"] == 0: + click.echo(muted(" no stored articles")) else: - click.echo( - f" uniqueness: {row['uniqueness']:.0%} " - f"gap-filling: {row['gap_filling']:.0%} " - f"{muted(f'({tagged} tagged article(s))')}" - ) + newest = row["latest"].date().isoformat() if row["latest"] else "undated" + click.echo(f" {row['articles']} article(s) {muted(f'newest {newest}')}") + if row["error_count"]: + click.echo(muted(f" {row['error_count']} fetch error(s)")) click.echo() @sources_command.command(name="show") @click.argument("name") def show_source(name): - """Show detailed calibration for a feed by exact or partial name.""" + """Show detail for a feed by exact or partial name.""" with get_db() as session: feed = session.query(RSSFeed).filter(RSSFeed.name.ilike(f"%{name}%")).first() if not feed: click.echo(muted(f"No feed matching '{name}'.")) return - cal = _compute_calibration(session).get(feed.id) - if cal is None: - click.echo(muted(f"No calibration available for {feed.name}.")) + row = _feed_stats(session).get(feed.id) + if row is None: + click.echo(muted(f"No detail available for {feed.name}.")) return click.echo(header(f"SOURCE: {feed.name}")) click.echo("=" * 80) - click.echo(f" uniqueness: {cal['uniqueness']:.0%}") - click.echo(f" gap-filling: {cal['gap_filling']:.0%}") - click.echo(f" tagged articles: {cal['tagged_articles']}") + click.echo(f" url: {feed.url}") + click.echo(f" category: {row['category'] or 'none'}") + click.echo(f" active: {'yes' if row['is_active'] else 'no'}") + click.echo(f" stored articles: {row['articles']}") + newest = row["latest"].date().isoformat() if row["latest"] else "none" + click.echo(f" newest article: {newest}") + fetched = row["last_fetched"].isoformat(sep=" ") if row["last_fetched"] else "never" + click.echo(f" last fetched: {fetched}") + if row["error_count"]: + click.echo(f" fetch errors: {row['error_count']}") + if row["last_error"]: + click.echo(muted(f" last error: {row['last_error']}")) click.echo() - - rows = ( - session.query(NarrativeFrame.label, func.count(ArticleFrame.id)) - .select_from(ArticleFrame) - .join(NarrativeFrame, ArticleFrame.frame_id == NarrativeFrame.id) - .join(Article, ArticleFrame.article_id == Article.id) - .filter(Article.feed_id == feed.id) - .group_by(NarrativeFrame.label) - .order_by(func.count(ArticleFrame.id).desc()) - .all() - ) - if rows: - click.echo(header("Frames carried")) - for label, count in rows: - click.echo(f" {label} {muted(f'({count} article(s))')}") diff --git a/src/cli/stake.py b/src/cli/stake.py deleted file mode 100644 index 483a79d..0000000 --- a/src/cli/stake.py +++ /dev/null @@ -1,238 +0,0 @@ -""" -The operator's side of the ledger: stake a claim, then record the outcome. - -Every other ledger command reads. These two write, and they are the reason the -graph is a calibration instrument rather than a reading list -- the thing being -calibrated is the operator's judgment, and until these existed the operator had -no way to stake anything. - -Both commands are pure local database work: no API key, no network call. - -Added 2026-08-27 for backlog task 011. -""" - -from datetime import datetime - -import click - -from ..database.connection import get_db -from ..database.models import ( - PREDICTION_AUTHOR_OPERATOR, - PREDICTION_OUTCOME_NO, - PREDICTION_OUTCOME_YES, - PREDICTION_STATUS_CONTRADICTED, - PREDICTION_STATUS_OPEN, - PREDICTION_STATUS_TRIGGERED, - QUESTION_STATUS_OPEN, - Prediction, - Question, -) -from ..utils import utcnow -from .colors import accent, header, muted, success, warning - -# The failure this command exists to prevent, stated once so both the help text -# and the rejection message can quote it. -_WHY_DATE_REQUIRED = ( - "A claim with no resolution date can never come due, so it can never be " - "graded. That is how the ledger accumulated 25 predictions phrased " - "'X would signal Y' and graded none of them." -) - -_WHY_CONFIDENCE_REQUIRED = ( - "A stake with an unstated confidence is a non-commitment in a different " - "costume. There is deliberately no default." -) - -# yes/no is the operator's verdict; the status vocabulary the model's check -# pass already uses is reused so `predictions triggered|contradicted` and -# `forecast` keep working unchanged rather than growing a parallel view. -_OUTCOME_STATUS = { - PREDICTION_OUTCOME_YES: PREDICTION_STATUS_TRIGGERED, - PREDICTION_OUTCOME_NO: PREDICTION_STATUS_CONTRADICTED, -} - - -@click.command(name="predict") -@click.argument("question_id", type=int) -@click.argument("claim") -@click.option( - "--by", - "due_by", - type=click.DateTime(formats=["%Y-%m-%d"]), - default=None, - metavar="YYYY-MM-DD", - help=f"Required. When this claim resolves. {_WHY_DATE_REQUIRED}", -) -@click.option( - "--confidence", - "confidence", - type=float, - default=None, - metavar="0.0-1.0", - help=f"Required. Your stated probability. {_WHY_CONFIDENCE_REQUIRED}", -) -def predict_command(question_id, claim, due_by, confidence): - """ - Stake a falsifiable claim on a question, with a date and a confidence. - - Both --by and --confidence are rejected at entry when missing. Nothing is - stored on a rejection. - """ - text = (claim or "").strip() - if not text: - raise click.UsageError("A prediction needs a claim. Nothing was stored.") - - if due_by is None: - raise click.UsageError( - f"--by is required. {_WHY_DATE_REQUIRED} " - "Give a resolution date, e.g. --by 2026-12-31. Nothing was stored." - ) - - if confidence is None: - raise click.UsageError( - f"--confidence is required. {_WHY_CONFIDENCE_REQUIRED} " - "Give a probability, e.g. --confidence 0.7. Nothing was stored." - ) - - if not 0.0 <= confidence <= 1.0: - raise click.UsageError( - f"--confidence must be between 0.0 and 1.0; got {confidence}. Nothing was stored." - ) - - now = utcnow() - if due_by.date() < now.date(): - raise click.UsageError( - f"--by {due_by.date().isoformat()} is in the past. A claim that was " - "already resolvable when you made it is not a forecast. Nothing was stored." - ) - - with get_db() as session: - question = session.query(Question).filter_by(id=question_id).first() - if question is None: - raise click.UsageError( - f"Question Q{question_id} not found. Declare it first with " - "'questions add'. Nothing was stored." - ) - if question.status != QUESTION_STATUS_OPEN: - raise click.UsageError( - f"Q{question_id} is {question.status}, not open. Nothing was stored." - ) - - prediction = Prediction( - question_id=question.id, - observable_text=text, - # The trigger for an operator claim is the operator, on the date - # they named. Recorded explicitly so the column keeps meaning the - # same thing it does for model predictions: what makes this - # gradeable. - trigger_condition=(f"Operator judgement on or after {due_by.date().isoformat()}."), - made_at=now, - made_in_synthesis_id=None, - author=PREDICTION_AUTHOR_OPERATOR, - due_by=due_by, - confidence=float(confidence), - status=PREDICTION_STATUS_OPEN, - ) - session.add(prediction) - session.commit() - - days_out = (due_by.date() - now.date()).days - click.echo( - success( - f"Staked P{prediction.id} on Q{question.id} at " - f"{confidence:.0%}, resolving {due_by.date().isoformat()} ({days_out}d out)." - ) - ) - click.echo(muted(f" Claim: {text}")) - click.echo(muted(f" Bears on: {question.text}")) - click.echo(muted(f" Record the outcome with: resolve {prediction.id} --outcome yes|no")) - - -@click.command(name="resolve") -@click.argument("prediction_id", type=int) -@click.option( - "--outcome", - type=click.Choice([PREDICTION_OUTCOME_YES, PREDICTION_OUTCOME_NO]), - required=True, - help="Did the claim, as written, come true?", -) -@click.option( - "--note", - required=True, - help="What settled it. Required: an outcome with no reason is not calibration data.", -) -def resolve_command(prediction_id, outcome, note): - """ - Record the outcome of a prediction. You resolve; the tool does not. - - Auto-resolving from coverage is deliberately absent: a tool that grades its - operator's calls using the corpus that produced them is measuring agreement - with itself. - """ - now = utcnow() - with get_db() as session: - prediction = session.query(Prediction).filter_by(id=prediction_id).first() - if prediction is None: - raise click.UsageError(f"Prediction P{prediction_id} not found. Nothing was changed.") - - if prediction.status != PREDICTION_STATUS_OPEN: - when = ( - prediction.resolved_at.date().isoformat() if prediction.resolved_at else "unknown" - ) - verdict = f" as '{prediction.outcome}'" if prediction.outcome else "" - raise click.UsageError( - f"P{prediction.id} is already {prediction.status}{verdict} " - f"(recorded {when}). A resolved prediction is not editable. " - "Nothing was changed." - ) - - prediction.outcome = outcome - prediction.status = _OUTCOME_STATUS[outcome] - # Separate from due_by on purpose: how late you graded it is itself - # calibration data. - prediction.resolved_at = now - prediction.resolution_note = note - question_text = prediction.question.text if prediction.question else None - session.commit() - - click.echo(success(f"P{prediction.id} resolved: {outcome}.")) - click.echo(muted(f" Claim: {prediction.observable_text}")) - if question_text: - click.echo(muted(f" Bears on: {question_text}")) - click.echo(muted(f" Note: {note}")) - click.echo(muted(f" {_timing_phrase(prediction.due_by, now)}")) - - -def _timing_phrase(due_by: datetime | None, resolved_at: datetime) -> str: - """How the resolution date sits against the due date.""" - if due_by is None: - return f"Resolved {resolved_at.date().isoformat()} (no due date on record)." - delta = (resolved_at.date() - due_by.date()).days - stamp = f"Due {due_by.date().isoformat()}, resolved {resolved_at.date().isoformat()}" - if delta > 0: - return f"{stamp} -- {delta}d late." - if delta < 0: - return f"{stamp} -- {-delta}d early, before it came due." - return f"{stamp} -- on the day." - - -def render_due_prediction(p: Prediction, now: datetime) -> None: - """One line-block for a prediction awaiting the operator's verdict.""" - overdue = (now.date() - p.due_by.date()).days if p.due_by else None - when = p.due_by.date().isoformat() if p.due_by else "no due date" - lateness = f" ({overdue}d ago)" if overdue else "" - click.echo(f" {accent(f'P{p.id}')} {warning('due ' + when)}{muted(lateness)}") - click.echo(f" {p.observable_text}") - if p.confidence is not None: - click.echo(muted(f" You said: {p.confidence:.0%}")) - click.echo(muted(f" resolve {p.id} --outcome yes|no --note '...'")) - click.echo() - - -def due_predictions_header(count: int) -> None: - """Heading for the awaiting-verdict block, shared by `forecast --due`.""" - click.echo(header(f"AWAITING YOUR VERDICT ({count})")) - click.echo( - muted("Your claims whose resolution date has passed. You grade these; nothing else does.") - ) - click.echo() diff --git a/src/config/beats.py b/src/config/beats.py deleted file mode 100644 index d5a22d4..0000000 --- a/src/config/beats.py +++ /dev/null @@ -1,665 +0,0 @@ -""" -Beat configuration loading. - -A **beat** is a subject: a standing topic with its own sources. It is a -different shape from the person-shaped profile in ``config/user_profile.json`` -(which models a location, a profession and a civic context), and the two -coexist -- a beat never replaces the profile and is never validated against it. -``src/utils/profile_loader.py`` is deliberately not reused here. - -Source selection reuses the existing ``applicability`` tag vocabulary from -``config/feeds/*.json`` (``scope`` / ``geo_tags`` / ``domain_tags`` / -``specialty_tags``) rather than introducing a parallel selector: a beat source -declares which of those tags a feed must carry, and :class:`FeedMatcher` is the -one place feeds are read from disk. - -Added 2026-08-26 for backlog task 004. -""" - -from __future__ import annotations - -import json -from dataclasses import dataclass -from datetime import date, timedelta -from pathlib import Path -from typing import Any - -from .feed_matcher import Feed, FeedMatcher - -# Where beat definitions live, relative to the project root. -DEFAULT_BEATS_DIR = Path(__file__).parent.parent.parent / "config" / "beats" - -# Ingestion adapters a beat source may name. An unknown adapter is an error -# rather than a silently skipped source, so a beat written against a future -# adapter fails loudly on this version. -# -# "federal_register" added 2026-08-26 (backlog task 005) alongside -# src/sources/federal_register.py. HTML scraping adapters are deliberately -# still absent -- see SOURCES.md. -SUPPORTED_ADAPTERS = frozenset({"rss", "federal_register"}) - -# Render channels a beat may request. These mirror src/render/. -SUPPORTED_CHANNELS = frozenset({"terminal", "markdown", "html", "email"}) - -BEAT_KEYS = frozenset( - { - "name", - "description", - "sources", - "coverage", - "standing_questions", - "coverage_probes", - "channels", - } -) -REQUIRED_BEAT_KEYS = frozenset({"name", "sources"}) -SOURCE_KEYS = frozenset({"adapter", "feed_tags", "geo_tags", "scope"}) -REQUIRED_SOURCE_KEYS = frozenset({"adapter", "feed_tags"}) - -# Keys permitted on one coverage probe (backlog task 010, 2026-08-27). -PROBE_KEYS = frozenset({"date", "what", "terms", "any_of", "window_days"}) -REQUIRED_PROBE_KEYS = frozenset({"date", "what", "terms"}) - -# How far either side of a probe's date a report of it is still a report of it. -# Symmetric because coverage runs both ways: a rule is trailed before it lands -# and analysed for a week after. Fourteen days is the interval the FedRAMP miss -# was measured over -- "3 incidental mentions and none within two weeks". -DEFAULT_PROBE_WINDOW_DAYS = 14 - -# The least evidence a probe may rest on. A probe is a claim that a match means -# the beat saw a specific event, and one bare term cannot carry that claim: -# `FedRAMP` alone is matched by an AWS region-launch post. Two independent -# pieces of evidence -- two required terms, or one required term plus one -# `any_of` group -- is the floor, enforced here rather than left to the author's -# judgement, because a probe that passes on generic terms manufactures exactly -# the confidence this feature exists to remove. -MIN_PROBE_EVIDENCE = 2 - -# The only three things a beat may track, and the plural config key that -# declares each. The mapping is closed on purpose: any other key -- `people`, -# `officials`, `staff`, anything -- is a validation error rather than a -# silently ignored block, so the boundary cannot be reintroduced by convention. -COVERAGE_KINDS: dict[str, str] = { - "orgs": "org", - "programs": "program", - "document_types": "document_type", -} -ENTITY_KINDS = frozenset(COVERAGE_KINDS.values()) - -# Keys permitted on the long form of one coverage entry. -COVERAGE_ENTRY_KEYS = frozenset({"name", "aliases"}) - - -class BeatNotFound(FileNotFoundError): - """Raised when no beat file exists for the requested name.""" - - def __init__(self, name: str, beats_dir: Path, available: list[str]) -> None: - self.name = name - self.available = available - known = ", ".join(available) if available else "none" - super().__init__(f"No beat named '{name}' in {beats_dir}. Available beats: {known}") - - -class BeatValidationError(ValueError): - """Raised when a beat file exists but does not describe a usable beat.""" - - def __init__(self, path: Path, problem: str) -> None: - self.path = path - self.problem = problem - super().__init__(f"Invalid beat file {path}: {problem}") - - -@dataclass(frozen=True) -class BeatSource: - """ - One source declaration inside a beat. - - ``feed_tags`` is matched against a feed's ``domain_tags`` and - ``specialty_tags`` -- the "what is this feed about" families. ``geo_tags`` - and ``scope`` are optional narrowing constraints matched against the - families of the same name. Within a family the match is ANY; across - families it is ALL. An omitted family constrains nothing. - - ``adapter`` is the one family that is not optional and not a tag: a source - declaration selects sources of one ingestion kind. Added 2026-08-26 with - backlog task 005 -- before it, every configured source was RSS and the - field was decoration. - """ - - adapter: str - feed_tags: tuple[str, ...] - geo_tags: tuple[str, ...] = () - scope: tuple[str, ...] = () - - def matches(self, feed: Feed) -> bool: - """True when ``feed`` satisfies every family this source constrains.""" - if feed.adapter != self.adapter: - return False - subject_tags = set(feed.domain_tags) | set(feed.specialty_tags) - if not subject_tags & set(self.feed_tags): - return False - if self.geo_tags and not set(feed.geo_tags) & set(self.geo_tags): - return False - return not (self.scope and not set(feed.scope) & set(self.scope)) - - -@dataclass(frozen=True) -class CoverageEntity: - """ - One institution a beat tracks: an organization, a program, or a type of - document. - - ``kind`` is one of :data:`ENTITY_KINDS`. There is no person kind, and the - absence is the point -- personnel rotate while offices persist, so a name - goes dark on reassignment and the silence reads as inactivity, which is a - wrong answer that looks like a real one. ``name`` is the canonical form - used everywhere the entity is displayed or stored; ``aliases`` are the - other surface forms that count as the same entity. - """ - - kind: str - name: str - aliases: tuple[str, ...] = () - - @property - def key(self) -> str: - """Stable identity of this entity within a beat: ``kind:name``.""" - return f"{self.kind}:{self.name}" - - @property - def terms(self) -> tuple[str, ...]: - """ - Every surface form that counts as this entity, canonical name first. - - Deduplicated but order-preserving, so matching is deterministic - regardless of how the config repeated itself. - """ - seen: dict[str, None] = {} - for term in (self.name, *self.aliases): - seen.setdefault(term, None) - return tuple(seen) - - -@dataclass(frozen=True) -class CoverageProbe: - """ - One thing that actually happened, used to test whether the beat can see it. - - A probe is not a search. It is a claim of the form "an event of this - description occurred on this date, and any report of it would carry these - words" -- which makes an unmatched probe a statement about the beat's - sources rather than about the phrasing of a query. - - ``terms`` must **all** appear in the same article; each group in ``any_of`` - contributes one alternative that must appear. The two levels exist because - a single distinctive term is too weak to be evidence and a whole headline is - too brittle to survive a second outlet's phrasing. - - A term ending in ``*`` is a stem: ``reinstat*`` matches "reinstated" and - "reinstatement". Without the marker a term matches whole words only. The - marker is explicit rather than implied so that the widening is visible in - the config to whoever has to trust the result -- see - :mod:`src.context.coverage_probe` for the matching rules themselves. - """ - - date: date - what: str - terms: tuple[str, ...] - any_of: tuple[tuple[str, ...], ...] = () - window_days: int = DEFAULT_PROBE_WINDOW_DAYS - - @property - def window(self) -> tuple[date, date]: - """The inclusive date range in which a report of this event counts.""" - span = timedelta(days=self.window_days) - return self.date - span, self.date + span - - @property - def evidence_count(self) -> int: - """How many independent things this probe requires of an article.""" - return len(self.terms) + len(self.any_of) - - def describe(self) -> str: - """The probe's requirement, written the way the config wrote it.""" - parts = [" AND ".join(self.terms)] if self.terms else [] - parts.extend("(" + " OR ".join(group) + ")" for group in self.any_of) - return " AND ".join(parts) - - -@dataclass(frozen=True) -class BeatConfig: - """ - A loaded, validated beat definition. - - ``coverage`` is the raw block as written; ``entities`` is the same block - parsed into :class:`CoverageEntity` values, which is what the institutional - activity pass reads. - - ``standing_questions`` are the questions this beat declares it is watching, - whether or not any given run's coverage mentions them. They are read by - ``src/context/standing_agenda.py`` (backlog task 007). - - ``coverage_probes`` are known past events the beat is tested against by - ``insightweaver beat coverage`` (backlog task 010). They are the answer to - "can this beat reach its domain" that an article count cannot give. - """ - - name: str - description: str - sources: tuple[BeatSource, ...] - coverage: dict[str, Any] - standing_questions: tuple[str, ...] - channels: tuple[str, ...] - config_path: str - entities: tuple[CoverageEntity, ...] = () - coverage_probes: tuple[CoverageProbe, ...] = () - - def resolve_feeds(self, matcher: FeedMatcher | None = None) -> list[Feed]: - """ - Every configured feed this beat's sources select, deduplicated by URL - and ordered by descending ``relevance_score`` then name. - - This reads ``config/feeds/``, not the database: it answers "which - sources does this beat claim", not "which of them have articles". - """ - matcher = matcher or FeedMatcher() - selected: dict[str, Feed] = {} - for feed in matcher.all_feeds: - if any(source.matches(feed) for source in self.sources): - selected.setdefault(feed.url, feed) - return sorted(selected.values(), key=lambda f: (-f.relevance_score, f.name)) - - def resolve_feed_urls(self, matcher: FeedMatcher | None = None) -> list[str]: - """The URLs of :meth:`resolve_feeds`, in the same order.""" - return [feed.url for feed in self.resolve_feeds(matcher)] - - -def available_beats(beats_dir: Path | str | None = None) -> list[str]: - """Names of every beat file on disk, sorted.""" - directory = Path(beats_dir) if beats_dir is not None else DEFAULT_BEATS_DIR - if not directory.is_dir(): - return [] - return sorted(path.stem for path in directory.glob("*.json")) - - -def load_beat(name: str, beats_dir: Path | str | None = None) -> BeatConfig: - """ - Load and validate ``config/beats/.json``. - - Fails fast: a missing file raises :class:`BeatNotFound`, and any structural - problem raises :class:`BeatValidationError` naming the problem. A beat is - never partially loaded and never falls back to defaults for a field the - author got wrong. - """ - directory = Path(beats_dir) if beats_dir is not None else DEFAULT_BEATS_DIR - path = directory / f"{name}.json" - if not path.is_file(): - raise BeatNotFound(name, directory, available_beats(directory)) - - try: - with open(path, encoding="utf-8") as handle: - raw = json.load(handle) - except json.JSONDecodeError as exc: - raise BeatValidationError(path, f"not valid JSON ({exc})") - - if not isinstance(raw, dict): - raise BeatValidationError(path, "top level must be a JSON object") - - unknown = set(raw) - BEAT_KEYS - if unknown: - raise BeatValidationError(path, f"unknown key(s): {', '.join(sorted(unknown))}") - - missing = REQUIRED_BEAT_KEYS - set(raw) - if missing: - raise BeatValidationError(path, f"missing required key(s): {', '.join(sorted(missing))}") - - beat_name = raw["name"] - if not isinstance(beat_name, str) or not beat_name.strip(): - raise BeatValidationError(path, "'name' must be a non-empty string") - if beat_name != path.stem: - raise BeatValidationError( - path, f"'name' is '{beat_name}' but the file is named '{path.stem}.json'" - ) - - description = raw.get("description", "") - if not isinstance(description, str): - raise BeatValidationError(path, "'description' must be a string") - - sources = _parse_sources(path, raw["sources"]) - - coverage = raw.get("coverage", {}) - if not isinstance(coverage, dict): - raise BeatValidationError(path, "'coverage' must be an object") - entities = _parse_coverage(path, coverage) - - standing_questions = _parse_standing_questions(path, raw.get("standing_questions", [])) - - coverage_probes = _parse_coverage_probes(path, raw.get("coverage_probes", [])) - - channels = _parse_channels(path, raw.get("channels", ["terminal"])) - - return BeatConfig( - name=beat_name, - description=description, - sources=sources, - coverage=coverage, - standing_questions=tuple(standing_questions), - channels=channels, - config_path=str(path), - entities=entities, - coverage_probes=coverage_probes, - ) - - -def _parse_coverage_probes(path: Path, raw_probes: Any) -> tuple[CoverageProbe, ...]: - """ - Validate the ``coverage_probes`` block into :class:`CoverageProbe` values. - - Every rule here refuses a probe rather than repairing it. A probe is - evidence about whether the beat's sources reach its domain, and a probe - that was quietly loosened -- an unparseable date defaulted to today, a - single generic term accepted -- would answer the question with something - other than what the author wrote. - """ - if not isinstance(raw_probes, list): - raise BeatValidationError(path, "'coverage_probes' must be a list of objects") - - parsed: list[CoverageProbe] = [] - for index, entry in enumerate(raw_probes): - where = f"coverage_probes[{index}]" - if not isinstance(entry, dict): - raise BeatValidationError(path, f"{where} must be an object, got {entry!r}") - - unknown = set(entry) - PROBE_KEYS - if unknown: - supported = ", ".join(sorted(PROBE_KEYS)) - raise BeatValidationError( - path, - f"{where} has unknown key(s): {', '.join(sorted(unknown))} (supported: {supported})", - ) - missing = REQUIRED_PROBE_KEYS - set(entry) - if missing: - raise BeatValidationError( - path, f"{where} missing required key(s): {', '.join(sorted(missing))}" - ) - - raw_date = entry["date"] - if not isinstance(raw_date, str): - raise BeatValidationError(path, f"{where}.date must be a 'YYYY-MM-DD' string") - try: - when = date.fromisoformat(raw_date) - except ValueError: - raise BeatValidationError( - path, f"{where}.date is not a 'YYYY-MM-DD' date: {raw_date!r}" - ) - - what = entry["what"] - if not isinstance(what, str) or not what.strip(): - raise BeatValidationError( - path, f"{where}.what must be a non-empty description of the event" - ) - - terms = _parse_probe_terms(path, f"{where}.terms", entry["terms"]) - if not terms: - raise BeatValidationError(path, f"{where}.terms must not be empty") - - raw_any_of = entry.get("any_of", []) - if not isinstance(raw_any_of, list): - raise BeatValidationError(path, f"{where}.any_of must be a list of term lists") - any_of: list[tuple[str, ...]] = [] - for group_index, group in enumerate(raw_any_of): - alternatives = _parse_probe_terms( - path, f"{where}.any_of[{group_index}]", group, allow_bare_string=False - ) - if not alternatives: - raise BeatValidationError( - path, - f"{where}.any_of[{group_index}] must not be empty -- an empty group of " - f"alternatives is satisfied by nothing and would make the probe unmatchable", - ) - any_of.append(alternatives) - - window_days = entry.get("window_days", DEFAULT_PROBE_WINDOW_DAYS) - if not isinstance(window_days, int) or isinstance(window_days, bool) or window_days < 1: - raise BeatValidationError( - path, f"{where}.window_days must be a positive integer, got {window_days!r}" - ) - - probe = CoverageProbe( - date=when, - what=what.strip(), - terms=terms, - any_of=tuple(any_of), - window_days=window_days, - ) - if probe.evidence_count < MIN_PROBE_EVIDENCE: - raise BeatValidationError( - path, - f"{where} rests on a single term ({terms[0]!r}). A probe needs at least " - f"{MIN_PROBE_EVIDENCE} independent requirements -- another entry in 'terms', " - f"or an 'any_of' group -- because one term on its own is matched by articles " - f"that merely mention the subject, and a probe that passes on a passing " - f"mention is worse than no probe.", - ) - parsed.append(probe) - - return tuple(parsed) - - -def _parse_probe_terms( - path: Path, where: str, value: Any, allow_bare_string: bool = True -) -> tuple[str, ...]: - """Validate one list of probe terms, order-preserving and deduplicated.""" - if allow_bare_string and isinstance(value, str): - value = [value] - if not isinstance(value, list): - raise BeatValidationError(path, f"{where} must be a list of strings") - - seen: dict[str, None] = {} - for term in value: - if not isinstance(term, str) or not term.strip(): - raise BeatValidationError( - path, f"{where} must contain only non-empty strings, got {term!r}" - ) - stripped = term.strip() - if stripped.rstrip("*") == "": - raise BeatValidationError( - path, - f"{where} contains {term!r}, which is a stem marker with no stem and " - f"would match every article", - ) - seen.setdefault(stripped, None) - return tuple(seen) - - -def _parse_coverage(path: Path, coverage: dict[str, Any]) -> tuple[CoverageEntity, ...]: - """ - Validate the ``coverage`` block into :class:`CoverageEntity` values. - - A beat tracks institutions, not people. ``coverage`` holds orgs, programs - and document types; a ``people`` key is refused here rather than ignored so - the boundary is enforced by the loader and cannot be reintroduced by - convention. This repository is public and covers a domain the operator - works in: a per-person activity ledger would read as surveillance of - colleagues regardless of the mechanism, and offices are the better signal - anyway since personnel rotate. - - The `people` rejection is checked before the general unknown-key check so - that the specific reason is the one the author reads. - """ - if "people" in coverage: - raise BeatValidationError( - path, - "'coverage.people' is not supported: a beat tracks organizations, programs " - "and document types, never individuals. A named person may appear as an " - "attribute of a specific document, never as an accumulated record.", - ) - - unknown = set(coverage) - set(COVERAGE_KINDS) - if unknown: - supported = ", ".join(sorted(COVERAGE_KINDS)) - raise BeatValidationError( - path, - f"'coverage' has unknown key(s): {', '.join(sorted(unknown))} " - f"(supported: {supported}). A beat tracks organizations, programs and " - f"document types, never individuals.", - ) - - parsed: list[CoverageEntity] = [] - seen: set[str] = set() - for block, kind in COVERAGE_KINDS.items(): - raw_entries = coverage.get(block, []) - if not isinstance(raw_entries, list): - raise BeatValidationError(path, f"'coverage.{block}' must be a list") - for index, entry in enumerate(raw_entries): - entity = _parse_coverage_entry(path, f"coverage.{block}[{index}]", kind, entry) - if entity.key in seen: - raise BeatValidationError( - path, f"'coverage.{block}' declares '{entity.name}' more than once" - ) - seen.add(entity.key) - parsed.append(entity) - - return tuple(parsed) - - -def _parse_coverage_entry(path: Path, where: str, kind: str, entry: Any) -> CoverageEntity: - """ - Validate one coverage entry, in either the short or the long form. - - Short form is a bare string -- the canonical name, matched on its own. - Long form is ``{"name": ..., "aliases": [...]}``. Any other key is an - error: a coverage entry describes an institution and its surface forms, - and there is nothing else for it to carry. - """ - if isinstance(entry, str): - name = entry - aliases: tuple[str, ...] = () - elif isinstance(entry, dict): - unknown = set(entry) - COVERAGE_ENTRY_KEYS - if unknown: - supported = ", ".join(sorted(COVERAGE_ENTRY_KEYS)) - raise BeatValidationError( - path, - f"{where} has unknown key(s): {', '.join(sorted(unknown))} " - f"(supported: {supported})", - ) - if "name" not in entry: - raise BeatValidationError(path, f"{where} is missing required key 'name'") - name = entry["name"] - raw_aliases = entry.get("aliases", []) - if not isinstance(raw_aliases, list): - raise BeatValidationError(path, f"{where}.aliases must be a list of strings") - for alias in raw_aliases: - if not isinstance(alias, str) or not alias.strip(): - raise BeatValidationError( - path, f"{where}.aliases must contain only non-empty strings, got {alias!r}" - ) - aliases = tuple(alias.strip() for alias in raw_aliases) - else: - raise BeatValidationError( - path, f"{where} must be a string or an object with a 'name', got {entry!r}" - ) - - if not isinstance(name, str) or not name.strip(): - raise BeatValidationError(path, f"{where} must have a non-empty 'name'") - - return CoverageEntity(kind=kind, name=name.strip(), aliases=aliases) - - -def _parse_sources(path: Path, raw_sources: Any) -> tuple[BeatSource, ...]: - """Validate the ``sources`` block into :class:`BeatSource` values.""" - if not isinstance(raw_sources, list) or not raw_sources: - raise BeatValidationError(path, "'sources' must be a non-empty list") - - parsed: list[BeatSource] = [] - for index, entry in enumerate(raw_sources): - where = f"sources[{index}]" - if not isinstance(entry, dict): - raise BeatValidationError(path, f"{where} must be an object") - - unknown = set(entry) - SOURCE_KEYS - if unknown: - raise BeatValidationError( - path, f"{where} has unknown key(s): {', '.join(sorted(unknown))}" - ) - missing = REQUIRED_SOURCE_KEYS - set(entry) - if missing: - raise BeatValidationError( - path, f"{where} missing required key(s): {', '.join(sorted(missing))}" - ) - - adapter = entry["adapter"] - if adapter not in SUPPORTED_ADAPTERS: - supported = ", ".join(sorted(SUPPORTED_ADAPTERS)) - raise BeatValidationError( - path, f"{where} adapter '{adapter}' is not supported (supported: {supported})" - ) - - parsed.append( - BeatSource( - adapter=adapter, - feed_tags=_parse_tag_list(path, where, "feed_tags", entry["feed_tags"], True), - geo_tags=_parse_tag_list(path, where, "geo_tags", entry.get("geo_tags", []), False), - scope=_parse_tag_list(path, where, "scope", entry.get("scope", []), False), - ) - ) - - return tuple(parsed) - - -def _parse_standing_questions(path: Path, raw_questions: Any) -> tuple[str, ...]: - """ - Validate the ``standing_questions`` block into a tuple of question texts. - - A standing question is declared by a human, deliberately (added 2026-08-26 - for backlog task 007). This loader therefore never invents one and never - drops one it cannot parse. Duplicates are refused rather than silently - deduplicated: two identical declarations mean the author has lost track of - the agenda, and collapsing them would hide that. - """ - if not isinstance(raw_questions, list): - raise BeatValidationError(path, "'standing_questions' must be a list of strings") - - parsed: list[str] = [] - seen: set[str] = set() - for index, entry in enumerate(raw_questions): - where = f"standing_questions[{index}]" - if not isinstance(entry, str) or not entry.strip(): - raise BeatValidationError(path, f"{where} must be a non-empty string, got {entry!r}") - text = entry.strip() - key = text.casefold() - if key in seen: - raise BeatValidationError( - path, f"{where} duplicates an earlier standing question: {text!r}" - ) - seen.add(key) - parsed.append(text) - return tuple(parsed) - - -def _parse_tag_list( - path: Path, where: str, field: str, value: Any, required_non_empty: bool -) -> tuple[str, ...]: - """Validate one tag family into a tuple of non-empty strings.""" - if not isinstance(value, list): - raise BeatValidationError(path, f"{where}.{field} must be a list of strings") - if required_non_empty and not value: - raise BeatValidationError(path, f"{where}.{field} must not be empty") - for tag in value: - if not isinstance(tag, str) or not tag.strip(): - raise BeatValidationError( - path, f"{where}.{field} must contain only non-empty strings, got {tag!r}" - ) - return tuple(value) - - -def _parse_channels(path: Path, raw_channels: Any) -> tuple[str, ...]: - """Validate the ``channels`` block.""" - if not isinstance(raw_channels, list) or not raw_channels: - raise BeatValidationError(path, "'channels' must be a non-empty list") - for channel in raw_channels: - if channel not in SUPPORTED_CHANNELS: - supported = ", ".join(sorted(SUPPORTED_CHANNELS)) - raise BeatValidationError(path, f"unknown channel {channel!r} (supported: {supported})") - return tuple(raw_channels) diff --git a/src/context/__init__.py b/src/context/__init__.py deleted file mode 100644 index 99dd94e..0000000 --- a/src/context/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Context Engineering Module -Curates optimal token context for Claude API calls -""" - -from .claude_client import ClaudeClient -from .curator import ContextCurator -from .synthesizer import NarrativeSynthesizer - -__all__ = [ - "ContextCurator", - "ClaudeClient", - "NarrativeSynthesizer", -] diff --git a/src/context/beat_scope.py b/src/context/beat_scope.py deleted file mode 100644 index 3d72761..0000000 --- a/src/context/beat_scope.py +++ /dev/null @@ -1,325 +0,0 @@ -""" -Beat scoping for the commitment graph. - -The graph tables (``questions``, ``question_situations``, ``predictions``) -carry no ``beat_id`` column. A row's beat is *derived* instead: a synthesis -belongs to a beat when ``beat_runs`` says so, a Question belongs to a beat when -it appeared in one of that beat's syntheses, and a Prediction inherits the beat -of the Question it keys off. Two scopes therefore exist: - -* a **beat scope** -- rows reachable from that beat's runs, and -* the **default scope** -- rows that no beat run ever touched, which is the - person-profile brief the tool has always produced. - -With no ``beat_runs`` rows at all -- the state of every existing database -before this feature is used -- the default scope is the whole graph and every -helper here is an identity filter, which is what keeps ``insightweaver brief`` -byte-identical to its previous behaviour. - -See docs/CONCEPTS.md, "Beats", for why derivation was chosen over a column. -Added 2026-08-26 for backlog task 004. -""" - -from __future__ import annotations - -import logging - -from sqlalchemy import and_, func, inspect, or_, select, true -from sqlalchemy.orm import Session - -from ..config.beats import BeatConfig -from ..database.models import ( - Beat, - BeatRun, - BeatStandingQuestion, - Prediction, - Question, - QuestionSituation, -) -from ..utils import utcnow - -logger = logging.getLogger(__name__) - - -class BeatTablesMissing(RuntimeError): - """Raised when a beat run is attempted on a database that predates beats.""" - - -class BeatNotRecorded(LookupError): - """Raised when a name does not match any beat that has ever run.""" - - def __init__(self, name: str, known: list[str]) -> None: - self.name = name - self.known = known - available = ", ".join(known) if known else "none yet" - super().__init__(f"No beat named '{name}' has recorded a run. Beats with runs: {available}") - - -# Which migration creates each beat table, for the "you have not migrated" -# message. beat_standing_questions arrived later than the other two (2026-08-26, -# backlog task 007), so a database can legitimately have one and not the other. -_BEAT_TABLE_MIGRATIONS = { - Beat.__tablename__: "add_beats", - BeatRun.__tablename__: "add_beats", - BeatStandingQuestion.__tablename__: "add_standing_questions", -} - - -def require_beat_tables(bind) -> None: - """ - Fail if this database has no beats at all. - - On the write side: without ``beats``/``beat_runs`` a run would have nowhere - to be attributed, and an unattributed beat run is indistinguishable from - the default brief -- so it must not start. Without - ``beat_standing_questions`` a declared agenda would have nowhere to be - recorded, and a standing question that cannot be recorded must not be - silently skipped -- that is the failure this feature exists to prevent. On - the read side: there is no beat ledger to show, and an empty listing would - read as "this beat has nothing", which is a wrong answer rather than a - missing one. - """ - inspector = inspect(bind) - missing = [name for name in _BEAT_TABLE_MIGRATIONS if not inspector.has_table(name)] - if missing: - migrations = sorted({_BEAT_TABLE_MIGRATIONS[name] for name in missing}) - commands = " and ".join( - f"'python -m src.database.migrations.{module}'" for module in migrations - ) - raise BeatTablesMissing( - f"This database has no {' or '.join(missing)} table, so it cannot record " - f"beats yet. Run {commands} (or 'insightweaver brief setup') once, then retry." - ) - - -def ensure_beat(session: Session, config: BeatConfig) -> int: - """ - Get or create the ``beats`` row for a loaded beat config, returning its id. - - The config file stays authoritative for sources; this row exists so runs - can be attributed and so the beat keeps a stable id across config edits. - """ - row = session.query(Beat).filter(Beat.name == config.name).first() - if row is None: - row = Beat( - name=config.name, - description=config.description, - config_path=config.config_path, - ) - session.add(row) - session.flush() - logger.info(f"Registered new beat '{config.name}' as beat_id {row.id}") - else: - row.description = config.description - row.config_path = config.config_path - row.updated_at = utcnow() - return int(row.id) - - -def beat_synthesis_ids(beat_id: int): - """Select of every synthesis id attributed to one beat.""" - return select(BeatRun.synthesis_id).where( - BeatRun.beat_id == beat_id, - BeatRun.synthesis_id.is_not(None), - ) - - -def _any_beat_synthesis_ids(): - """Select of every synthesis id attributed to any beat.""" - return select(BeatRun.synthesis_id).where(BeatRun.synthesis_id.is_not(None)) - - -def _beat_runs_recorded(session: Session) -> bool: - """ - Whether this database can have attributed any run to a beat. - - A database predating this feature has no ``beat_runs`` table. That is not a - fallback case to paper over -- "the table does not exist" and "the table is - empty" are the same statement: no run has ever been attributed to a beat. - Both mean the default scope is the whole graph. Checking lets the default - brief keep working on an unmigrated database instead of failing on a table - it has no reason to care about. - - A beat *run* never takes this path: registering the beat writes to ``beats`` - first, so a missing table surfaces there, loudly, before any scoped query. - """ - return inspect(session.get_bind()).has_table(BeatRun.__tablename__) - - -def _standing_questions_recorded(session: Session) -> bool: - """ - Whether this database can hold declared standing questions. - - Same reasoning as :func:`_beat_runs_recorded`: a database that predates - backlog task 007 has no ``beat_standing_questions`` table, and "the table - does not exist" and "the table is empty" are the same statement -- no beat - has declared anything. - """ - return inspect(session.get_bind()).has_table(BeatStandingQuestion.__tablename__) - - -def question_scope_filter(session: Session, beat_id: int | None): - """ - A criterion restricting a ``Question`` query to one scope. - - A question is in a beat's scope when it appeared in one of that beat's - syntheses **or** when that beat declared it as a standing question. The - second clause is what keeps a declared question that no coverage has yet - touched inside the beat rather than in the default scope: it has no - situation links to derive a beat from, so without it the person brief's - matcher could bind to a compliance beat's agenda item. - - ``beat_id=None`` yields the default scope: questions that have never - appeared in any beat's synthesis and that no beat has declared. A question - with no links at all (one created earlier in this same transaction) is in - the default scope, which is harmless because such a question is not a match - candidate anyway. - """ - runs_possible = _beat_runs_recorded(session) - standing_possible = _standing_questions_recorded(session) - - if beat_id is None and not runs_possible and not standing_possible: - return true() - - if beat_id is None: - excluded = [] - if runs_possible: - excluded.append( - Question.id.not_in( - select(QuestionSituation.question_id).where( - QuestionSituation.synthesis_id.in_(_any_beat_synthesis_ids()) - ) - ) - ) - if standing_possible: - excluded.append(Question.id.not_in(select(BeatStandingQuestion.question_id))) - return and_(*excluded) - - included = [ - Question.id.in_( - select(QuestionSituation.question_id).where( - QuestionSituation.synthesis_id.in_(beat_synthesis_ids(beat_id)) - ) - ) - ] - if standing_possible: - included.append( - Question.id.in_( - select(BeatStandingQuestion.question_id).where( - BeatStandingQuestion.beat_id == beat_id - ) - ) - ) - return or_(*included) - - -def prediction_scope_filter(session: Session, beat_id: int | None): - """ - A criterion restricting a ``Prediction`` query to one scope. - - Predictions key off Questions, so scoping the questions scopes the ledger: - a beat run grades only its own open observables and never expires or - resolves another scope's. - """ - if beat_id is None and not _beat_runs_recorded(session): - return true() - - in_scope_questions = select(Question.id).where(question_scope_filter(session, beat_id)) - return Prediction.question_id.in_(in_scope_questions) - - -def scoped_appearance_count(session: Session, question_id: int | None, beat_id: int | None) -> int: - """ - How many times a question has already appeared *within one scope*. - - This is what makes ``Q47 (run 4, asked 2026-03-12)`` mean "the fourth time - this beat raised it" rather than "the fourth time anything raised it". - """ - if question_id is None: - return 0 - - stmt = ( - select(func.count()) - .select_from(QuestionSituation) - .where(QuestionSituation.question_id == question_id) - ) - if beat_id is None: - if _beat_runs_recorded(session): - stmt = stmt.where(QuestionSituation.synthesis_id.not_in(_any_beat_synthesis_ids())) - else: - stmt = stmt.where(QuestionSituation.synthesis_id.in_(beat_synthesis_ids(beat_id))) - - return int(session.execute(stmt).scalar() or 0) - - -def beat_id_by_name(session: Session, name: str) -> int: - """ - Resolve a beat name to its id for the read side. - - Deliberately keyed off the ``beats`` table rather than ``config/beats/``: - a ledger you have already accumulated stays readable after its config file - is edited or deleted. Reading is about what ran, not about what is - currently configured to run. - """ - require_beat_tables(session.get_bind()) - row = session.query(Beat).filter(Beat.name == name).first() - if row is None: - known = [entry[0] for entry in session.query(Beat.name).order_by(Beat.name).all()] - raise BeatNotRecorded(name, known) - return int(row.id) - - -def declared_standing_question_ids(session: Session, beat_id: int | None) -> set[int]: - """ - The Question ids one beat has declared as standing questions. - - ``beat_id=None`` returns an empty set: the default brief is a person, not a - subject, and only a beat can declare an agenda. - """ - if beat_id is None or not _standing_questions_recorded(session): - return set() - - rows = ( - session.query(BeatStandingQuestion.question_id) - .filter(BeatStandingQuestion.beat_id == beat_id) - .all() - ) - return {int(entry[0]) for entry in rows} - - -def owning_beat_names(session: Session, question_id: int | None) -> list[str]: - """ - Which beats, if any, a question has appeared under or been declared by. - - Used by the id-addressed commands to disclose the scope a row belongs to. - An empty list means the question belongs to the default scope. A declared - standing question that has never moved still names its beat here, because - it belongs to that ledger from the moment it is declared. - """ - if question_id is None: - return [] - - names: set[str] = set() - - if _beat_runs_recorded(session): - appeared = ( - session.query(Beat.name) - .join(BeatRun, BeatRun.beat_id == Beat.id) - .join(QuestionSituation, QuestionSituation.synthesis_id == BeatRun.synthesis_id) - .filter(QuestionSituation.question_id == question_id) - .distinct() - .all() - ) - names.update(entry[0] for entry in appeared) - - if _standing_questions_recorded(session): - declared = ( - session.query(Beat.name) - .join(BeatStandingQuestion, BeatStandingQuestion.beat_id == Beat.id) - .filter(BeatStandingQuestion.question_id == question_id) - .distinct() - .all() - ) - names.update(entry[0] for entry in declared) - - return sorted(names) diff --git a/src/context/cross_cluster_reconciler.py b/src/context/cross_cluster_reconciler.py deleted file mode 100644 index 06d953e..0000000 --- a/src/context/cross_cluster_reconciler.py +++ /dev/null @@ -1,104 +0,0 @@ -""" -Cross-Cluster Reconciler - -Pass 3: after all situations finalize, look across them for meta-fractures -- -a single underlying frame conflict appearing in multiple topically distinct -situations. This catches frame structure as a corpus property rather than -just a per-cluster one. -""" - -import logging - -from ..prompts.meta_frames import META_FRAME_RECONCILIATION_PROMPT -from ._json import parse_claude_json -from .claude_client import ClaudeClient - -logger = logging.getLogger(__name__) - -RECONCILER_MODEL = "claude-haiku-4-5-20251001" -RECONCILER_MAX_TOKENS = 2048 -# Reconciliation is only meaningful with at least two situations to compare. -MIN_SITUATIONS = 2 - - -class CrossClusterReconciler: - """Detects meta-fractures spanning multiple situations.""" - - def __init__(self, client: ClaudeClient | None = None): - self.client = client or ClaudeClient(model=RECONCILER_MODEL) - - async def reconcile(self, situations: list[dict]) -> list[dict]: - """ - Return the list of detected meta-fractures, validated. - - Each entry has ``name``, ``description``, ``situation_indices`` - (subset of valid indices, length >= 2), and ``shared_point``. Returns - an empty list when there is nothing to reconcile, the LLM fails, or - no meta-fractures are found. - """ - if len(situations) < MIN_SITUATIONS: - return [] - - situations_block = self._build_situations_block(situations) - if not situations_block: - return [] - - prompt = META_FRAME_RECONCILIATION_PROMPT.format(situations_block=situations_block) - - try: - raw = await self.client.analyze( - system_prompt=( - "You find meta-fractures across multiple frame analyses. " - "Be conservative; empty results are common." - ), - user_message=prompt, - effort="low", - max_tokens=RECONCILER_MAX_TOKENS, - ) - except Exception as e: - logger.warning(f"Reconciler LLM call failed; returning no meta-fractures: {e}") - return [] - - parsed = parse_claude_json(raw, label="reconciler response") - result: list[dict] = [] - n = len(situations) - for entry in parsed.get("meta_fractures", []): - if not isinstance(entry, dict): - continue - name = str(entry.get("name", "")).strip() - description = str(entry.get("description", "")).strip() - shared_point = str(entry.get("shared_point", "")).strip() - indices_raw = entry.get("situation_indices", []) - if not isinstance(indices_raw, list): - continue - indices = sorted({i for i in indices_raw if isinstance(i, int) and 0 <= i < n}) - if len(indices) < MIN_SITUATIONS or not name or not shared_point: - continue - result.append( - { - "name": name, - "description": description, - "situation_indices": indices, - "shared_point": shared_point, - } - ) - return result - - @staticmethod - def _build_situations_block(situations: list[dict]) -> str: - """Render each situation's frame info compactly for the prompt.""" - lines: list[str] = [] - for i, situation in enumerate(situations): - title = situation.get("title", "Untitled") - frame = situation.get("coverage_frame") or {} - narrative_layers = (frame.get("narrative_layers") or "").strip() - fractures = (frame.get("fractures") or "").strip() - if not narrative_layers and not fractures: - continue - block = [f"[{i}] {title}"] - if narrative_layers: - block.append(f" layers: {narrative_layers}") - if fractures: - block.append(f" fractures: {fractures}") - lines.append("\n".join(block)) - return "\n\n".join(lines) diff --git a/src/context/curator.py b/src/context/curator.py deleted file mode 100644 index 6279c20..0000000 --- a/src/context/curator.py +++ /dev/null @@ -1,461 +0,0 @@ -""" -Context Curator -Builds optimal context for Claude API calls from database and user profile -Implements token budget management following Anthropic's context engineering guidance -""" - -import json -import logging -from datetime import timedelta -from typing import Any - -from sqlalchemy import select -from sqlalchemy.orm import Session - -from ..config.beats import BeatConfig -from ..database.connection import get_db -from ..database.models import Article, NarrativeSynthesis, RSSFeed -from ..utils import utcnow -from ..utils.profile_loader import UserProfile, get_user_profile -from ..utils.profiler import profile -from .topic_matcher import TopicMatcher - -logger = logging.getLogger(__name__) - - -class ContextCurator: - """Curates optimal context for Claude analysis with token budget management""" - - # Token budget allocation for Claude Sonnet 4 (200k context window) - TOKEN_BUDGET = { - "system_prompt": 5000, # Perspective + framework + examples - "articles": 50000, # Article content (50 articles × ~1000 tokens) - "historical": 10000, # Compressed memory/patterns - "response": 8000, # Claude's output - "safety_margin": 127000, # Reserved buffer - } - - # Total available: 200,000 tokens - # Allocated: 73,000 tokens (system + articles + historical + response) - # Safety margin: 127,000 tokens - - def __init__( - self, - user_profile: UserProfile | None = None, - topic_filters: dict | None = None, - beat: BeatConfig | None = None, - ): - """ - Initialize context curator - - Args: - user_profile: User profile for personalization - perspective_id: Perspective to use for analysis framing (defaults to user preference or daily_intelligence_brief) - topic_filters: Optional topic/scope filters dict (e.g., {'topics': ['cybersecurity'], 'scopes': ['local']}) - beat: Optional beat. When given, article selection is restricted to - that beat's configured sources and nothing else. The user - profile is still used to frame the analysis -- a beat scopes - which sources are read, not who the brief is for. - """ - try: - self.user_profile = user_profile or get_user_profile() - except FileNotFoundError: - logger.warning("User profile not found, context will not be personalized") - self.user_profile = None - - # Topic filtering - self.topic_filters = topic_filters or {} - self.topic_matcher = TopicMatcher() if self.topic_filters else None - - # Beat scoping. Resolved once here so the feed set a run used is fixed - # for that run even if config/beats/ changes mid-flight. - self.beat = beat - self.beat_feed_urls: list[str] = beat.resolve_feed_urls() if beat else [] - if beat: - logger.info(f"Beat '{beat.name}' selects {len(self.beat_feed_urls)} configured feed(s)") - if not self.beat_feed_urls: - logger.warning( - f"Beat '{beat.name}' matched no feeds in config/feeds/. " - f"Its brief will have no articles to analyze." - ) - - async def curate_for_narrative_synthesis( - self, hours: int = 48, max_articles: int = 50 - ) -> dict[str, Any]: - """ - Curate context for narrative synthesis with token budget enforcement - - Args: - hours: Hours to look back for articles - max_articles: Maximum articles to include - - Returns: - Curated context dictionary with token metadata - """ - with profile("CONTEXT_CURATION_TOTAL"): - with get_db() as session: - # Get recent unfiltered articles - with profile("DB_QUERY_ARTICLES"): - articles = self._get_recent_articles(session, hours, max_articles) - - # Get historical context from prior syntheses - with profile("DB_QUERY_SYNTHESES"): - memory = self._get_historical_memory(session) - - # Format articles while session is still active (prevents DetachedInstanceError) - formatted_articles = self._format_articles(articles) - - # Build initial context - context = { - "user_profile": self._format_user_profile(), - "articles": formatted_articles, - "memory": memory, - "instructions": self._get_synthesis_instructions(), - } - - # Enforce token budget - context = self._enforce_token_budget(context) - - return context - - def curate_for_summary( - self, articles: list[Article], brief_type: str = "daily" - ) -> dict[str, Any]: - """ - Curate context for executive summary generation - - Args: - articles: Articles to summarize - brief_type: Type of brief ("daily" or "weekly") - - Returns: - Curated context dictionary - """ - context = { - "user_profile": self._format_user_profile(), - "articles": self._format_articles(articles[:20]), # Top 20 - "instructions": self._get_summary_instructions(brief_type), - } - - return context - - def _get_recent_articles( - self, session: Session, hours: int, max_articles: int - ) -> list[Article]: - """ - Get recent unfiltered articles from database with optional topic filtering - - Args: - session: Database session - hours: Hours to look back - max_articles: Maximum articles to return - - Returns: - List of articles (filtered by topic/scope if filters provided) - """ - cutoff_time = utcnow() - timedelta(hours=hours) - - # Base query: recent, unfiltered articles - query = session.query(Article).filter( - Article.fetched_at >= cutoff_time, Article.filtered.is_(False) - ) - - # Beat scoping. This is the hard boundary that makes a beat's brief - # "drawn only from that beat's sources": articles from any other feed - # are excluded before topic filters or limits are considered. A beat - # whose feeds are not yet in rss_feeds simply yields nothing, rather - # than silently falling back to the full corpus. - if self.beat is not None: - beat_feed_ids = select(RSSFeed.id).where(RSSFeed.url.in_(self.beat_feed_urls)) - query = query.filter(Article.feed_id.in_(beat_feed_ids)) - - # If no topic filters, use existing logic - if not self.topic_filters: - articles = query.order_by(Article.fetched_at.desc()).limit(max_articles).all() - - scope_note = f"beat '{self.beat.name}'" if self.beat else "no filters" - logger.info(f"Curated {len(articles)} articles from last {hours} hours ({scope_note})") - return articles - - # Fetch 2x candidates for filtering buffer - candidate_articles = query.order_by(Article.fetched_at.desc()).limit(max_articles * 2).all() - - logger.info( - f"Fetched {len(candidate_articles)} candidate articles for filtering " - f"with filters: {self.topic_filters}" - ) - - # Apply topic filters using TopicMatcher - if self.topic_matcher and self.user_profile: - filtered_articles = self.topic_matcher.filter_articles( - articles=candidate_articles, - topic_filters=self.topic_filters, - user_profile=self.user_profile, - ) - else: - logger.warning("TopicMatcher not initialized or user_profile missing, skipping filters") - filtered_articles = candidate_articles - - # Limit to max_articles after filtering - final_articles = filtered_articles[:max_articles] - - # Warn if too few matches - if len(final_articles) == 0: - logger.warning( - f"No articles matched filters {self.topic_filters} in last {hours} hours. " - f"Consider expanding time window or using fewer filters." - ) - elif len(final_articles) < 5: - logger.warning( - f"Only {len(final_articles)} articles matched filters {self.topic_filters}. " - f"Consider broadening filter criteria." - ) - - logger.info( - f"Curated {len(final_articles)} articles from {len(candidate_articles)} candidates " - f"(last {hours} hours, filters: {self.topic_filters})" - ) - - return final_articles - - def _get_historical_memory(self, session: Session) -> str: - """ - Get enhanced historical context from past syntheses - - Includes: - - Last 5 synthesis summaries (increased from 3) - - Tracked trends and predictions - - Identified patterns - """ - # Get last 5 narrative syntheses (increased from 3) - syntheses = ( - session.query(NarrativeSynthesis) - .order_by(NarrativeSynthesis.generated_at.desc()) - .limit(5) - .all() - ) - - if not syntheses: - return "No historical context available." - - memory_parts = [""] - memory_parts.append("## Recent Intelligence Summaries") - memory_parts.append("Previous analyses for trend tracking and prediction verification:\n") - - for i, synth in enumerate(syntheses, 1): - date_str = synth.generated_at.strftime("%Y-%m-%d") - - # Use bottom_line summary if available (new format), fallback to executive_summary (old format) - if synth.synthesis_data: - try: - import json - - data = ( - json.loads(synth.synthesis_data) - if isinstance(synth.synthesis_data, str) - else synth.synthesis_data - ) - bottom_line = data.get("bottom_line", {}) - summary = bottom_line.get("summary", "") - - if summary: - memory_parts.append(f"\n**{date_str} ({i} days ago):**") - memory_parts.append(summary) - - # Include immediate actions if present - actions = bottom_line.get("immediate_actions", []) - if actions: - memory_parts.append( - " Key actions identified: " + "; ".join(actions[:2]) - ) - elif synth.executive_summary: - # Fallback to old format - memory_parts.append(f"\n**{date_str}:** {synth.executive_summary[:400]}...") - except Exception: - # If parsing fails, use executive_summary - if synth.executive_summary: - memory_parts.append(f"\n**{date_str}:** {synth.executive_summary[:400]}...") - elif synth.executive_summary: - memory_parts.append(f"\n**{date_str}:** {synth.executive_summary[:400]}...") - - memory_parts.append("\n## Analysis Continuity") - memory_parts.append("Use the above summaries to:") - memory_parts.append("• Identify continuing trends vs new developments") - memory_parts.append("• Verify or update previous predictions") - memory_parts.append("• Note unusual patterns (topic appearing/disappearing)") - memory_parts.append("• Provide comparative context (e.g., 'unlike last week when...')") - memory_parts.append("") - - return "\n".join(memory_parts) - - def _enforce_token_budget(self, context: dict[str, Any]) -> dict[str, Any]: - """ - Enforce token budget by compressing context if needed - - Strategy: - 1. Count current tokens - 2. If over budget, compress in this order: - - Reduce articles (50 → 30 → 20) - - Trim historical memory (keep top 2 summaries) - - Truncate article content (1000 → 500 tokens) - - Args: - context: Raw context dictionary - - Returns: - Token-optimized context dictionary - """ - # Estimate current token usage - tokens = self._estimate_tokens(context) - - logger.info(f"Initial context: ~{tokens['total']} tokens") - - # Check if we're over budget (system + articles + historical) - budget_used = tokens["system"] + tokens["articles"] + tokens["historical"] - budget_limit = ( - self.TOKEN_BUDGET["system_prompt"] - + self.TOKEN_BUDGET["articles"] - + self.TOKEN_BUDGET["historical"] - ) - - if budget_used <= budget_limit: - logger.info(f"Within token budget: {budget_used}/{budget_limit} tokens") - context["_token_metadata"] = tokens - return context - - logger.warning(f"Over token budget: {budget_used}/{budget_limit} tokens, compressing...") - - # Compression step 1: Reduce article count - if len(context["articles"]) > 30: - logger.info(f"Reducing articles from {len(context['articles'])} to 30") - context["articles"] = context["articles"][:30] - tokens = self._estimate_tokens(context) - budget_used = tokens["system"] + tokens["articles"] + tokens["historical"] - - if budget_used <= budget_limit: - context["_token_metadata"] = tokens - return context - - # Compression step 2: Further reduce articles - if len(context["articles"]) > 20: - logger.info(f"Further reducing articles from {len(context['articles'])} to 20") - context["articles"] = context["articles"][:20] - tokens = self._estimate_tokens(context) - budget_used = tokens["system"] + tokens["articles"] + tokens["historical"] - - if budget_used <= budget_limit: - context["_token_metadata"] = tokens - return context - - # Compression step 3: Trim historical memory - if "Recent Intelligence Summaries" in context["memory"]: - logger.info("Trimming historical memory to last 2 summaries") - memory_lines = context["memory"].split("\n") - # Keep header + first 2 summaries (approximately) - context["memory"] = "\n".join(memory_lines[:8]) - tokens = self._estimate_tokens(context) - - context["_token_metadata"] = tokens - logger.info(f"Compressed context: ~{tokens['total']} tokens") - return context - - def _estimate_tokens(self, context: dict[str, Any]) -> dict[str, int]: - """ - Estimate token count for context components - - Uses rough approximation: 1 token ≈ 4 characters - This is conservative for English text - - Args: - context: Context dictionary - - Returns: - Token count breakdown - """ - - def count_chars(obj): - return len(json.dumps(obj)) if obj else 0 - - system_chars = count_chars(context.get("user_profile", {})) + len( - context.get("instructions", "") - ) - - articles_chars = count_chars(context.get("articles", [])) - memory_chars = len(context.get("memory", "")) - - return { - "system": system_chars // 4, - "articles": articles_chars // 4, - "historical": memory_chars // 4, - "total": (system_chars + articles_chars + memory_chars) // 4, - } - - def _format_user_profile(self) -> dict[str, Any]: - """Format user profile for context""" - if not self.user_profile: - return {"location": "Unknown", "professional_domains": [], "civic_interests": []} - - location = self.user_profile.get_primary_location() - # Handle both string and dict formats for primary_location - if isinstance(location, str): - location_str = location - elif isinstance(location, dict): - location_str = f"{location.get('city', '')}, {location.get('state', '')}" - else: - location_str = "Unknown" - - return { - "location": location_str, - "professional_domains": self.user_profile.get_professional_domains(), - "civic_interests": self.user_profile.get_civic_interests(), - } - - def _format_articles(self, articles: list[Article]) -> list[dict[str, Any]]: - """Format articles for context inclusion""" - formatted = [] - - for article in articles: - # Use embedding_summary if available, fallback to normalized_content - content = ( - article.embedding_summary or article.normalized_content or article.description or "" - ) - - formatted.append( - { - "id": article.id, # Include ID for tracking - "title": article.title, - "source": article.feed.name if article.feed else "Unknown", - "published_date": article.published_date.isoformat() - if article.published_date - else None, - "content": content, - "url": article.url, - "entities": article.entities if article.entities else [], - } - ) - - return formatted - - def _get_synthesis_instructions(self) -> str: - """Get instructions for narrative synthesis with user context.""" - if self.user_profile: - location = self.user_profile.get_primary_location() - professional = self.user_profile.get_professional_context() - - if isinstance(location, dict): - location_str = f"{location.get('city', '')}, {location.get('state', '')}" - else: - location_str = str(location) if location else "not specified" - - professional_domains = ", ".join(professional.get("professional_domains", [])) - else: - location_str = "not specified" - professional_domains = "not specified" - - return f"""## User Context - -Location: {location_str} -Professional domains: {professional_domains} - -Analyze the provided articles and generate a structured intelligence brief. -Use the user's location and professional context to assess relevance and implications.""" diff --git a/src/context/decision_router.py b/src/context/decision_router.py deleted file mode 100644 index d569a05..0000000 --- a/src/context/decision_router.py +++ /dev/null @@ -1,158 +0,0 @@ -""" -Decision Router -Runs after situation synthesis: matches each situation against the user's -open decision factors and produces evidence specs. This is the only place -situation-to-decision routing happens, so the chain from coverage to -decision stays centralized and inspectable. -""" - -import logging -from dataclasses import dataclass - -from sqlalchemy.orm import Session - -from ..database.models import ( - DECISION_STATUS_OPEN, - EVIDENCE_DIRECTION_COMPLICATES, - EVIDENCE_DIRECTION_NEUTRAL, - EVIDENCE_DIRECTION_SUPPORTS, - Decision, - DecisionFactor, -) -from ..prompts.decisions import DECISION_ROUTING_PROMPT -from ._json import parse_claude_json -from .claude_client import ClaudeClient - -logger = logging.getLogger(__name__) - -ROUTER_MODEL = "claude-haiku-4-5-20251001" -ROUTER_MAX_TOKENS = 4096 -SITUATION_EXCERPT_CHARS = 600 - -_VALID_DIRECTIONS = { - EVIDENCE_DIRECTION_SUPPORTS, - EVIDENCE_DIRECTION_COMPLICATES, - EVIDENCE_DIRECTION_NEUTRAL, -} -_VALID_EPISTEMIC = {"reported_fact", "single_source", "consensus", "speculation"} - - -@dataclass -class RoutedEvidence: - """One validated routing result, ready for the synthesizer to persist.""" - - situation_index: int - decision_id: int - factor_id: int - direction: str - epistemic_status: str - excerpt: str - - -class DecisionRouter: - """Routes situation evidence into the user's open decision factors.""" - - def __init__(self, client: ClaudeClient | None = None): - self.client = client or ClaudeClient(model=ROUTER_MODEL) - - async def route_evidence( - self, situations: list[dict], session: Session - ) -> list[RoutedEvidence]: - """ - Match situations against open decision factors. - - Returns validated RoutedEvidence specs. Does not write to the - database -- the synthesizer attaches synthesis_id and question_id - and persists them. Returns an empty list when there are no open - decisions or no situations. - """ - if not situations: - return [] - - factors = ( - session.query(DecisionFactor) - .join(Decision, DecisionFactor.decision_id == Decision.id) - .filter(Decision.status == DECISION_STATUS_OPEN) - .all() - ) - if not factors: - return [] - - # factor_id -> decision_id, for validating LLM output and attaching - # the decision to each routed evidence row. - decision_by_factor = {f.id: f.decision_id for f in factors} - - raw = await self._llm_route(factors, situations) - results: list[RoutedEvidence] = [] - for entry in raw: - sit_idx = entry.get("situation_index") - factor_id = entry.get("factor_id") - if not isinstance(sit_idx, int) or not (0 <= sit_idx < len(situations)): - continue - if not isinstance(factor_id, int) or factor_id not in decision_by_factor: - continue - - direction = entry.get("direction") - if direction not in _VALID_DIRECTIONS: - direction = EVIDENCE_DIRECTION_NEUTRAL - epistemic = entry.get("epistemic_status") - if epistemic not in _VALID_EPISTEMIC: - epistemic = "speculation" - excerpt = str(entry.get("excerpt", "")).strip() - if not excerpt: - continue - - results.append( - RoutedEvidence( - situation_index=sit_idx, - decision_id=decision_by_factor[factor_id], - factor_id=factor_id, - direction=direction, - epistemic_status=epistemic, - excerpt=excerpt, - ) - ) - return results - - async def _llm_route(self, factors: list[DecisionFactor], situations: list[dict]) -> list[dict]: - """One Haiku call: returns the raw evidence list (unvalidated).""" - factor_lines = [] - for f in factors: - decision_name = f.decision.name if f.decision else "(unknown decision)" - update_clause = f.what_would_update_me or "(no update rule set)" - factor_lines.append( - f"[{f.id}] decision: {decision_name}\n" - f" factor: {f.name}\n" - f" would update me: {update_clause}" - ) - factors_block = "\n\n".join(factor_lines) - - situation_lines = [] - for i, situation in enumerate(situations): - title = situation.get("title", "Untitled") - narrative = (situation.get("narrative") or "")[:SITUATION_EXCERPT_CHARS].strip() - situation_lines.append(f"[{i}] {title}\n {narrative}") - situations_block = "\n\n".join(situation_lines) - - prompt = DECISION_ROUTING_PROMPT.format( - factors_block=factors_block, - situations_block=situations_block, - ) - - try: - raw = await self.client.analyze( - system_prompt=( - "You route news analysis into a user's standing decisions. " - "Be conservative; most situations route nowhere." - ), - user_message=prompt, - effort="low", - max_tokens=ROUTER_MAX_TOKENS, - ) - except Exception as e: - logger.warning(f"Decision router LLM call failed; routing nothing: {e}") - return [] - - parsed = parse_claude_json(raw, label="decision router response") - evidence = parsed.get("evidence", []) - return evidence if isinstance(evidence, list) else [] diff --git a/src/context/frame_manager.py b/src/context/frame_manager.py deleted file mode 100644 index 8711768..0000000 --- a/src/context/frame_manager.py +++ /dev/null @@ -1,324 +0,0 @@ -""" -Frame Manager -Handles frame discovery, matching, and persistence for the narrative frame glossary. -Called by the synthesizer between Pass 1 (clustering) and Pass 2 (situation synthesis). -""" - -import logging -from typing import Any - -from ..database.connection import get_db -from ..database.models import ArticleFrame, FrameGap, NarrativeFrame, TopicCluster -from ..prompts.frames import ( - FRAME_AWARE_SYNTHESIS_PROMPT, - FRAME_CLASSIFICATION_PROMPT, - FRAME_DISCOVERY_PROMPT, -) -from ._json import parse_claude_json -from .claude_client import ClaudeClient - -logger = logging.getLogger(__name__) - -# Haiku for the cheap article-to-frame classification pass. -CLASSIFIER_MODEL = "claude-haiku-4-5-20251001" -CLASSIFIER_MAX_TOKENS = 2048 -CLASSIFIER_SNIPPET_CHARS = 300 - - -class FrameManager: - """Manages the narrative frame glossary lifecycle.""" - - def __init__(self, client: ClaudeClient): - self.client = client - - def find_matching_cluster(self, cluster: dict) -> TopicCluster | None: - """ - Find an existing TopicCluster that matches this cluster by keyword overlap. - - Returns the matching TopicCluster or None. - """ - keywords = {kw.lower() for kw in cluster.get("keywords", [])} - title_words = set(cluster.get("title", "").lower().split()) - match_terms = keywords | title_words - - if not match_terms: - return None - - try: - with get_db() as session: - existing_clusters = session.query(TopicCluster).all() - - best_match = None - best_overlap = 0 - - for tc in existing_clusters: - tc_keywords = {kw.lower() for kw in (tc.keywords or [])} - tc_name_words = set(tc.name.lower().split()) - tc_terms = tc_keywords | tc_name_words - - overlap = len(match_terms & tc_terms) - if overlap > best_overlap and overlap >= 2: - best_overlap = overlap - best_match = tc - - if best_match: - # Detach from session before returning - session.expunge(best_match) - - return best_match - - except Exception as e: - logger.error(f"Error finding matching cluster: {e}") - return None - - def get_validated_frames(self, topic_cluster_id: int) -> list[NarrativeFrame]: - """Get validated frames for a topic cluster.""" - try: - with get_db() as session: - frames = ( - session.query(NarrativeFrame) - .filter_by(topic_cluster_id=topic_cluster_id, validated=True) - .all() - ) - session.expunge_all() - return frames - except Exception as e: - logger.error(f"Error getting validated frames: {e}") - return [] - - def get_cluster_frames(self, topic_cluster_id: int) -> list[NarrativeFrame]: - """Get all frames (validated or not) for a topic cluster.""" - try: - with get_db() as session: - frames = ( - session.query(NarrativeFrame).filter_by(topic_cluster_id=topic_cluster_id).all() - ) - session.expunge_all() - return frames - except Exception as e: - logger.error(f"Error getting cluster frames: {e}") - return [] - - async def classify_articles_to_frames( - self, cluster_articles: list[dict], frames: list[NarrativeFrame] - ) -> int: - """ - Tag each article in a cluster with the frame it most exhibits and - write ArticleFrame rows. Populating this mapping is what lets the - diet view show which frames each feed carries. - - Returns the number of ArticleFrame rows written. - """ - if not cluster_articles or not frames: - return 0 - - # label -> frame id, for resolving the model's frame_label output. - frame_id_by_label = {f.label.strip().lower(): f.id for f in frames} - - frames_block = "\n".join( - f"- {f.label}: {f.description or '(no description)'}" for f in frames - ) - article_lines = [] - indexed_article_ids: list[int | None] = [] - for i, article in enumerate(cluster_articles): - title = article.get("title", "Untitled") - content = article.get("content") or article.get("description") or "" - snippet = content[:CLASSIFIER_SNIPPET_CHARS].strip() - article_lines.append(f"[{i}] {title}\n {snippet}") - indexed_article_ids.append(article.get("id")) - articles_block = "\n\n".join(article_lines) - - prompt = FRAME_CLASSIFICATION_PROMPT.format( - frames_block=frames_block, articles_block=articles_block - ) - - classifier = ClaudeClient(model=CLASSIFIER_MODEL) - try: - raw = await classifier.analyze( - system_prompt="You tag articles with the narrative frame they most exhibit.", - user_message=prompt, - effort="low", - max_tokens=CLASSIFIER_MAX_TOKENS, - ) - except Exception as e: - logger.warning(f"Frame classification call failed; skipping: {e}") - return 0 - - parsed = parse_claude_json(raw, label="frame classification response") - written = 0 - try: - with get_db() as session: - for entry in parsed.get("classifications", []): - idx = entry.get("article_index") - label = entry.get("frame_label") - confidence = entry.get("confidence") - if not isinstance(idx, int) or not (0 <= idx < len(indexed_article_ids)): - continue - article_id = indexed_article_ids[idx] - if article_id is None or not isinstance(label, str): - continue - frame_id = frame_id_by_label.get(label.strip().lower()) - if frame_id is None: - continue - if not isinstance(confidence, int | float): - confidence = 0.5 - session.add( - ArticleFrame( - article_id=article_id, - frame_id=frame_id, - confidence=max(0.0, min(1.0, float(confidence))), - ) - ) - written += 1 - session.commit() - except Exception as e: - logger.error(f"Failed to store article-frame classifications: {e}") - return 0 - - return written - - def build_frame_aware_prompt(self, topic_name: str, frames: list[NarrativeFrame]) -> str: - """ - Build FRAME_AWARE_SYNTHESIS_PROMPT for injection into situation synthesis. - - Returns the formatted prompt string to append to the synthesis task. - """ - frames_yaml = "" - for frame in frames: - frames_yaml += f"- **{frame.label}**\n" - if frame.description: - frames_yaml += f" Description: {frame.description}\n" - if frame.assumptions: - frames_yaml += f" Assumes: {frame.assumptions}\n" - frames_yaml += "\n" - - return FRAME_AWARE_SYNTHESIS_PROMPT.format( - topic_name=topic_name, - frames_yaml=frames_yaml, - ) - - async def discover_frames( - self, cluster_articles: list[dict], cluster: dict - ) -> dict[str, Any] | None: - """ - Run frame discovery on a cluster with no known frames. - - Returns the parsed frame discovery result or None on failure. - """ - # Build article text for the prompt - article_texts = [] - for i, article in enumerate(cluster_articles, 1): - title = article.get("title", "Untitled") - content = article.get("content", article.get("description", "")) - # Truncate content to keep context manageable - snippet = content[:500] if content else "" - article_texts.append(f"[{i}] {title}\n{snippet}") - - articles_text = "\n\n".join(article_texts) - prompt = FRAME_DISCOVERY_PROMPT.format(articles=articles_text) - - try: - response = await self.client.analyze( - system_prompt="You are a media frame analyst. Identify narrative frames in news coverage.", - user_message=prompt, - effort="low", - ) - return parse_claude_json(response, label="frame discovery response") - except Exception as e: - logger.error(f"Frame discovery failed for '{cluster.get('title', '')}': {e}") - return None - - def store_discovered_frames(self, cluster: dict, discovery_result: dict) -> int | None: - """ - Store frame discovery results in the database. - - Creates a TopicCluster and unvalidated NarrativeFrame records. - Also creates FrameGap records for absent frames. - - Returns the TopicCluster ID or None on failure. - """ - try: - with get_db() as session: - # Create topic cluster - tc = TopicCluster( - name=cluster.get("title", "Unknown"), - keywords=cluster.get("keywords", []), - ) - session.add(tc) - session.flush() - - # Store present frames as unvalidated candidates - for frame_data in discovery_result.get("frames_present", []): - frame = NarrativeFrame( - topic_cluster_id=tc.id, - label=frame_data.get("label", "Unknown"), - description=frame_data.get("emphasizes", ""), - assumptions=frame_data.get("assumed_premise", ""), - validated=False, - ) - session.add(frame) - - # Store absent frames as gaps - for absent in discovery_result.get("frames_absent", []): - # Find matching feed recommendation - feed_suggestion = "" - for rec in discovery_result.get("gap_recommendations", []): - if rec.get("absent_frame_label") == absent.get("label"): - feed_suggestion = rec.get("feed_type", "") - break - - gap = FrameGap( - topic_cluster_id=tc.id, - frame_label=absent.get("label", "Unknown"), - occurrences=1, - feed_suggestion=feed_suggestion, - ) - session.add(gap) - - session.commit() - logger.info( - f"Stored frame discovery for '{tc.name}': " - f"{len(discovery_result.get('frames_present', []))} frames, " - f"{len(discovery_result.get('frames_absent', []))} gaps" - ) - return tc.id - - except Exception as e: - logger.error(f"Failed to store frame discovery: {e}") - return None - - def update_frame_gaps(self, topic_cluster_id: int, absent_frame_labels: list[str]): - """ - Increment gap occurrences for known frames that are absent today. - - Called when FRAME_AWARE_SYNTHESIS_PROMPT identifies absent known frames. - """ - if not absent_frame_labels: - return - - try: - with get_db() as session: - for label in absent_frame_labels: - existing_gap = ( - session.query(FrameGap) - .filter_by( - topic_cluster_id=topic_cluster_id, - frame_label=label, - ) - .first() - ) - - if existing_gap: - existing_gap.occurrences += 1 - else: - gap = FrameGap( - topic_cluster_id=topic_cluster_id, - frame_label=label, - occurrences=1, - ) - session.add(gap) - - session.commit() - - except Exception as e: - logger.error(f"Failed to update frame gaps: {e}") diff --git a/src/context/institutional_activity.py b/src/context/institutional_activity.py deleted file mode 100644 index 01234ac..0000000 --- a/src/context/institutional_activity.py +++ /dev/null @@ -1,306 +0,0 @@ -""" -Institutional activity: what a beat's declared institutions did this run, -against what they usually do. - -The signal is the **delta**, never the count. "FedRAMP PMO: 6" is noise -- the -PMO appears most days and a flat tally reproduces that fact every morning -without adding anything. "FedRAMP PMO appeared in 6 items this run, against a -trailing average of 1" is the observation worth a brief line. Everything in -this module exists to turn the first into the second. - -Three rules follow from that, and each is load-bearing: - -* **Zeroes are recorded.** A run in which an office said nothing is an - observation about that office. Omitting it would average only over the days - something happened, and the average would always say "normal". -* **Silence is reported when it is unusual.** An entity that has been active - and is quiet appears, because the absence is the finding. Dropping it is the - same class of bug as a standing question vanishing on a quiet day. -* **An entity nobody has ever mentioned does not appear at all.** Declaring an - entity is a hypothesis about where news comes from; a hypothesis that has - never paid out is a note about the config, not a line in the brief. - -A count is an observation, not a verdict. Nothing here scores, ranks or -interprets: no "most active", no significance, no ordering by magnitude. The -tool's north star is that no entity stores a truth value, and activity is not -importance. - -Added 2026-08-26 for backlog task 006. -""" - -from __future__ import annotations - -import logging -from collections.abc import Iterable, Sequence -from dataclasses import dataclass -from typing import Any - -from sqlalchemy.orm import Session - -from ..config.beats import CoverageEntity -from ..database.models import ENTITY_KINDS, BeatEntity, EntityMention -from ..utils import utcnow -from .entity_matcher import compile_entities, count_item_mentions - -logger = logging.getLogger(__name__) - -__all__ = [ - "MOVEMENT_DOWN", - "MOVEMENT_FIRST_RUN", - "MOVEMENT_UNCHANGED", - "MOVEMENT_UP", - "TRAILING_WINDOW", - "ActivityObservation", - "EntityActivity", - "observe_activity", - "record_mentions", - "sync_entities", -] - -# How many previous recorded runs the trailing average is taken over. Short on -# purpose: a beat runs daily and a month-long baseline would smother a real -# move under history nobody remembers. -TRAILING_WINDOW = 5 - -# What counts as movement. A change must clear both bars: at least one whole -# item (so an average of 0.2 drifting to 0 is not "news"), and at least half -# the baseline (so 12 against an average of 11 is not "news" either). -MIN_ABSOLUTE_MOVE = 1.0 -MIN_RELATIVE_MOVE = 0.5 - -MOVEMENT_UP = "up" -MOVEMENT_DOWN = "down" -MOVEMENT_UNCHANGED = "unchanged" -MOVEMENT_FIRST_RUN = "first_run" - - -class UnsupportedEntityKind(ValueError): - """ - Raised when something other than an org, program or document type reaches - the write path. - - The loader already refuses a ``coverage.people`` key, so this cannot happen - through configuration. It is checked again here because this is the last - point before a row is written, and a row is the thing that outlives the - run. A caller that has to route around this check is asking for the design - the task forbids. - """ - - -@dataclass(frozen=True) -class EntityActivity: - """One entity's reading for one run: this run's count against its baseline.""" - - kind: str - name: str - count: int - trailing_average: float | None # None on the first run an entity is observed - prior_runs: int # how many recorded runs the average is taken over - movement: str - - def as_dict(self) -> dict[str, Any]: - """The shape stored in synthesis metadata and read by the renderers.""" - return { - "kind": self.kind, - "name": self.name, - "count": self.count, - "trailing_average": self.trailing_average, - "prior_runs": self.prior_runs, - "movement": self.movement, - } - - -@dataclass(frozen=True) -class ActivityObservation: - """ - One run's institutional activity: what to render, and what to write. - - ``entities`` is the rendered view -- only entities with something to say. - ``counts_by_entity_id`` is the ledger view -- every declared entity, - zeroes included, so the baseline stays honest. - """ - - entities: tuple[EntityActivity, ...] - counts_by_entity_id: dict[int, int] - items_scanned: int - never_observed: int # declared entities with no mentions and no history - - def as_dict(self) -> dict[str, Any]: - """The ``metadata["institutional_activity"]`` payload.""" - return { - "window": TRAILING_WINDOW, - "items_scanned": self.items_scanned, - "never_observed": self.never_observed, - "entities": [entity.as_dict() for entity in self.entities], - } - - -def sync_entities( - session: Session, beat_id: int, entities: Iterable[CoverageEntity] -) -> dict[str, int]: - """ - Get or create the ``beat_entities`` rows for a beat's coverage block. - - Returns ``{CoverageEntity.key: beat_entities.id}``. Aliases are refreshed - from the config on every run, because the config is authoritative for what - an entity is called; the row exists only to give its history somewhere to - hang. Entities dropped from the config keep their rows and their past - counts -- deleting them would silently rewrite the record of what was - observed. - """ - ids: dict[str, int] = {} - for entity in entities: - if entity.kind not in ENTITY_KINDS: - raise UnsupportedEntityKind( - f"Cannot record a '{entity.kind}' entity: a beat tracks " - f"{', '.join(ENTITY_KINDS)} and nothing else." - ) - row = ( - session.query(BeatEntity) - .filter( - BeatEntity.beat_id == beat_id, - BeatEntity.kind == entity.kind, - BeatEntity.name == entity.name, - ) - .first() - ) - if row is None: - row = BeatEntity( - beat_id=beat_id, - kind=entity.kind, - name=entity.name, - aliases=list(entity.aliases), - ) - session.add(row) - session.flush() - logger.info(f"Registered beat entity '{entity.name}' ({entity.kind}) as id {row.id}") - elif list(row.aliases or []) != list(entity.aliases): - row.aliases = list(entity.aliases) - row.updated_at = utcnow() - ids[entity.key] = int(row.id) - return ids - - -def _trailing_counts(session: Session, entity_id: int) -> list[int]: - """ - This entity's last :data:`TRAILING_WINDOW` recorded counts, most recent - first. Zero rows included -- see the module docstring. - """ - rows: list[Any] = ( - session.query(EntityMention.item_count) - .filter(EntityMention.entity_id == entity_id) - .order_by(EntityMention.observed_at.desc(), EntityMention.id.desc()) - .limit(TRAILING_WINDOW) - .all() - ) - return [int(row[0] or 0) for row in rows] - - -def classify_movement(count: int, trailing_average: float | None) -> str: - """ - How this run's count sits against the baseline. - - Returns :data:`MOVEMENT_FIRST_RUN` when there is no baseline yet -- on day - one everything is a first run and the section is honestly uninformative, - which is expected and must not be tuned away. - """ - if trailing_average is None: - return MOVEMENT_FIRST_RUN - delta = count - trailing_average - threshold = max(MIN_ABSOLUTE_MOVE, MIN_RELATIVE_MOVE * trailing_average) - if delta >= threshold: - return MOVEMENT_UP - if -delta >= threshold: - return MOVEMENT_DOWN - return MOVEMENT_UNCHANGED - - -def observe_activity( - session: Session, - beat_id: int, - entities: Sequence[CoverageEntity], - texts: Sequence[str], -) -> ActivityObservation: - """ - Count this run's mentions and read them against each entity's baseline. - - Reads history and registers entities; writes no mention rows. Those are - written by :func:`record_mentions` once the run has a ``beat_runs`` id, so - a run that fails after this point leaves no half-recorded baseline behind. - """ - entity_ids = sync_entities(session, beat_id, entities) - counts = count_item_mentions(compile_entities(entities), texts) - - readings: list[EntityActivity] = [] - counts_by_entity_id: dict[int, int] = {} - never_observed = 0 - - for entity in entities: - count = counts.get(entity.key, 0) - entity_id = entity_ids[entity.key] - counts_by_entity_id[entity_id] = count - - history = _trailing_counts(session, entity_id) - trailing_average = (sum(history) / len(history)) if history else None - - # "Has this ever been seen" is asked of the counts, not of the rows: - # every entity gets a row every run, so the presence of rows says only - # that the beat has run before. - ever_observed = count > 0 or any(history) - if not ever_observed: - never_observed += 1 - continue - - readings.append( - EntityActivity( - kind=entity.kind, - name=entity.name, - count=count, - trailing_average=( - round(trailing_average, 2) if trailing_average is not None else None - ), - prior_runs=len(history), - movement=classify_movement(count, trailing_average), - ) - ) - - # Alphabetical within kind, never by count: ordering by magnitude would - # make this a leaderboard, and activity is not significance. - readings.sort(key=lambda reading: (ENTITY_KINDS.index(reading.kind), reading.name.lower())) - - return ActivityObservation( - entities=tuple(readings), - counts_by_entity_id=counts_by_entity_id, - items_scanned=len(texts), - never_observed=never_observed, - ) - - -def record_mentions( - session: Session, - counts_by_entity_id: dict[int, int], - *, - beat_run_id: int | None, - synthesis_id: int | None, - items_scanned: int | None = None, -) -> int: - """ - Write this run's counts, one row per entity including the zeroes. - - Added to the caller's session without committing: the mention rows belong - to the same transaction as the synthesis and the ``beat_runs`` row, so a - stored run is either fully recorded or does not exist. - """ - observed_at = utcnow() - for entity_id, count in sorted(counts_by_entity_id.items()): - session.add( - EntityMention( - entity_id=entity_id, - beat_run_id=beat_run_id, - synthesis_id=synthesis_id, - item_count=count, - items_scanned=items_scanned, - observed_at=observed_at, - ) - ) - return len(counts_by_entity_id) diff --git a/src/context/prediction_tracker.py b/src/context/prediction_tracker.py deleted file mode 100644 index 48bb6e8..0000000 --- a/src/context/prediction_tracker.py +++ /dev/null @@ -1,175 +0,0 @@ -""" -Prediction Tracker -Runs before each synthesis: expires stale open predictions, then grades the -rest against today's coverage. This is what makes the tool's own -forward-looking statements auditable -- every observable it flagged gets -checked against what later actually showed up in the feeds. -""" - -import logging -from datetime import timedelta - -from sqlalchemy.orm import Session - -from ..database.models import ( - PREDICTION_EXPIRY_DAYS, - PREDICTION_STATUS_CONTRADICTED, - PREDICTION_STATUS_EXPIRED, - PREDICTION_STATUS_OPEN, - PREDICTION_STATUS_TRIGGERED, - Prediction, -) -from ..prompts.predictions import PREDICTION_CHECK_PROMPT -from ..utils import utcnow -from ._json import parse_claude_json -from .beat_scope import prediction_scope_filter -from .claude_client import ClaudeClient - -logger = logging.getLogger(__name__) - -TRACKER_MODEL = "claude-haiku-4-5-20251001" -OPEN_PREDICTION_LIMIT = 200 -CHECK_MAX_TOKENS = 4096 -# Article snippets passed to the check pass; titles + a short lead is enough -# to tell whether an observable was reported. -COVERAGE_SNIPPET_CHARS = 200 - - -class PredictionTracker: - """Grades the open-prediction ledger against fresh coverage.""" - - def __init__(self, client: ClaudeClient | None = None): - self.client = client or ClaudeClient(model=TRACKER_MODEL) - - async def check_open_predictions( - self, articles: list[dict], session: Session, beat_id: int | None = None - ) -> dict: - """ - Expire stale predictions, then grade the rest against today's coverage. - - Returns a summary dict for the brief's transparency block. Writes - status changes to the session but does not commit -- the caller owns - the transaction. - - ``beat_id`` restricts both the expiry sweep and the grading pass to one - scope's ledger, so a beat run never resolves or ages out the default - brief's observables and vice versa. With no beat runs on record the - scope is the whole ledger, i.e. the behaviour that predates beats. - """ - now = utcnow() - scope = prediction_scope_filter(session, beat_id) - summary = { - "checked": 0, - "triggered": 0, - "contradicted": 0, - "expired": 0, - "still_open": 0, - } - - # Expire anything that aged out without resolution. - cutoff = now - timedelta(days=PREDICTION_EXPIRY_DAYS) - stale = ( - session.query(Prediction) - .filter( - Prediction.status == PREDICTION_STATUS_OPEN, - Prediction.made_at < cutoff, - scope, - ) - .all() - ) - for pred in stale: - pred.status = PREDICTION_STATUS_EXPIRED - pred.resolved_at = now - pred.resolution_note = ( - f"Expired after {PREDICTION_EXPIRY_DAYS} days without resolution." - ) - summary["expired"] = len(stale) - - open_preds = ( - session.query(Prediction) - .filter(Prediction.status == PREDICTION_STATUS_OPEN, scope) - .order_by(Prediction.made_at.desc()) - .limit(OPEN_PREDICTION_LIMIT) - .all() - ) - summary["checked"] = len(open_preds) - summary["still_open"] = len(open_preds) - - if not open_preds or not articles: - session.flush() - return summary - - verdicts = await self._llm_check(open_preds, articles) - for pred in open_preds: - verdict = verdicts.get(pred.id) - if not verdict: - continue - label, note = verdict - if label == "triggered": - pred.status = PREDICTION_STATUS_TRIGGERED - pred.resolved_at = now - pred.resolution_note = f"Triggered: {note}" - summary["triggered"] += 1 - summary["still_open"] -= 1 - elif label == "contradicted": - pred.status = PREDICTION_STATUS_CONTRADICTED - pred.resolved_at = now - pred.resolution_note = f"Contradicted: {note}" - summary["contradicted"] += 1 - summary["still_open"] -= 1 - - session.flush() - return summary - - async def _llm_check( - self, predictions: list[Prediction], articles: list[dict] - ) -> dict[int, tuple[str, str]]: - """One Haiku call: maps prediction id to (verdict, note).""" - pred_lines = [] - for p in predictions: - question_text = p.question.text if p.question else "(unknown question)" - pred_lines.append( - f"[{p.id}] observable: {p.observable_text}\n" - f" trigger: {p.trigger_condition}\n" - f" bears on: {question_text}" - ) - predictions_block = "\n\n".join(pred_lines) - - coverage_lines = [] - for article in articles: - title = article.get("title", "Untitled") - content = article.get("content") or article.get("description") or "" - snippet = content[:COVERAGE_SNIPPET_CHARS].strip() - source = article.get("source", "Unknown") - coverage_lines.append(f"- {title} ({source})\n {snippet}") - coverage_block = "\n".join(coverage_lines) - - prompt = PREDICTION_CHECK_PROMPT.format( - predictions_block=predictions_block, - coverage_block=coverage_block, - ) - - try: - raw = await self.client.analyze( - system_prompt=( - "You grade open predictions against fresh news coverage. " - "Be conservative; 'open' is the default." - ), - user_message=prompt, - effort="low", - max_tokens=CHECK_MAX_TOKENS, - ) - except Exception as e: - logger.warning(f"Prediction check LLM call failed; leaving all open: {e}") - return {} - - parsed = parse_claude_json(raw, label="prediction check response") - results: dict[int, tuple[str, str]] = {} - valid = {"triggered", "contradicted", "open"} - for entry in parsed.get("verdicts", []): - pid = entry.get("prediction_id") - verdict = entry.get("verdict") - note = entry.get("note", "") - if isinstance(pid, int) and verdict in valid: - results[pid] = (verdict, str(note)) - return results diff --git a/src/context/question_matcher.py b/src/context/question_matcher.py deleted file mode 100644 index d220613..0000000 --- a/src/context/question_matcher.py +++ /dev/null @@ -1,295 +0,0 @@ -""" -Question Matcher -Resolves freshly-proposed unresolved_questions against the persistent -Question graph: matches to existing open questions where appropriate, -creates new Question records otherwise. -""" - -import logging -import re -import unicodedata -from dataclasses import dataclass - -from sqlalchemy.orm import Session - -from ..database.models import ( - QUESTION_STATUS_OPEN, - QUESTION_STATUS_RESOLVED, - Question, -) -from ..prompts.questions import QUESTION_MATCHING_PROMPT -from ..utils import utcnow -from ._json import parse_claude_json -from .beat_scope import declared_standing_question_ids, question_scope_filter -from .claude_client import ClaudeClient - -logger = logging.getLogger(__name__) - -MATCHER_MODEL = "claude-haiku-4-5-20251001" -OPEN_QUESTION_LIMIT = 100 -MATCHER_MAX_TOKENS = 2048 - -# How much subject vocabulary a proposed question must share with a *declared* -# standing question before the matcher will bind them (added 2026-08-26, -# backlog task 007). -# -# Emergent questions are matched on the LLM's judgement alone, and that stays -# exactly as it was. A declared question is different in kind: it was written -# by a human before any coverage existed, it is typically broad ("Which CSPs -# move to FedRAMP authorized?"), and broad text is what a similarity matcher -# over-matches. Binding it to unrelated coverage is worse than missing a real -# match, because a standing question that falsely reads "moved" destroys the -# one thing the agenda is for -- so the LLM's answer is treated as a proposal -# and this lexical gate has a veto over it. -# -# Measured as overlap against the *shorter* question's content tokens rather -# than Jaccard: the two texts are routinely different lengths (a declared -# question is terser than a question drawn out of a week's coverage), and -# Jaccard penalises that length difference rather than the topic drift the -# gate is actually trying to catch. -STANDING_BINDING_MIN_OVERLAP = 0.6 - -# Function words carry no subject, so they would inflate every overlap score. -# fmt: off -_STOPWORDS = frozenset( - [ - "a", "an", "and", "any", "are", "as", "at", "be", "been", "being", "but", "by", "can", - "could", "did", "do", "does", "doing", "for", "from", "had", "has", "have", "how", "i", - "if", "in", "into", "is", "it", "its", "may", "might", "must", "of", "on", "or", - "over", "should", "so", "some", "such", "than", "that", "the", "their", "them", "then", - "there", "these", "they", "this", "those", "to", "under", "up", "was", "were", "what", - "when", "where", "whether", "which", "while", "who", "whom", "why", "will", "with", - "within", "would", "you", "your", - ] -) -# fmt: on - - -_PUNCT = re.compile(r"[^\w\s]+") -_WHITESPACE = re.compile(r"\s+") - - -def normalize_question(text: str) -> str: - """Lowercase, strip diacritics and punctuation, collapse whitespace.""" - if not text: - return "" - folded = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("ascii") - stripped = _PUNCT.sub(" ", folded.lower()) - return _WHITESPACE.sub(" ", stripped).strip() - - -def content_tokens(text: str) -> set[str]: - """ - The subject-bearing words of a question: normalized, stopwords removed, - trailing plural ``s`` folded. - - The plural fold is naive on purpose -- it is not stemming, it just stops - "deadline inside 90 days" and "a 90-day deadline" from reading as different - subjects. Both sides of a comparison go through it, so it can only merge - tokens, never invent an overlap that the words do not support. - """ - tokens = set() - for token in normalize_question(text).split(): - if token in _STOPWORDS: - continue - if len(token) > 3 and token.endswith("s") and not token.endswith("ss"): - token = token[:-1] - tokens.add(token) - return tokens - - -def binding_overlap(proposed: str, candidate: str) -> float: - """ - How much subject vocabulary two questions share, from 0.0 to 1.0. - - The overlap coefficient: shared content tokens over the size of the smaller - token set. Returns 0.0 when either question has no content tokens at all, - which refuses the bind rather than guessing at it. - """ - left, right = content_tokens(proposed), content_tokens(candidate) - if not left or not right: - return 0.0 - return len(left & right) / min(len(left), len(right)) - - -@dataclass -class ProposedQuestion: - """One question proposed by today's synthesis.""" - - text: str - is_primary: bool = True - - -class QuestionMatcher: - """Resolves proposed questions against persistent Question records.""" - - def __init__(self, client: ClaudeClient | None = None): - self.client = client or ClaudeClient(model=MATCHER_MODEL) - - async def resolve_questions( - self, - proposed: list[ProposedQuestion], - session: Session, - beat_id: int | None = None, - ) -> list[Question]: - """ - Map each proposed question to a Question row (existing or new). - - Returns a list parallel to ``proposed``: each Question is either an - existing row or a newly added one (session.add). The caller commits. - - ``beat_id`` selects the scope questions are matched within: a beat's - run only ever binds to that beat's questions, and the default (no-beat) - run only ever binds to questions no beat has claimed. Without any beat - runs on record the scope is the whole graph, i.e. the behaviour that - predates beats. - - Declared standing questions live in the same scope and enter the same - matcher, but bind on a tighter rule -- see - :data:`STANDING_BINDING_MIN_OVERLAP`. Emergent questions are unaffected. - """ - if not proposed: - return [] - - scope = question_scope_filter(session, beat_id) - standing_ids = declared_standing_question_ids(session, beat_id) - normalized = [normalize_question(p.text) for p in proposed] - resolved: list[Question | None] = [None] * len(proposed) - - for i, norm in enumerate(normalized): - if not norm: - continue - hit = ( - session.query(Question) - .filter( - Question.status == QUESTION_STATUS_OPEN, - Question.normalized_text == norm, - scope, - ) - .first() - ) - if hit: - resolved[i] = hit - - remaining_idx = [i for i, r in enumerate(resolved) if r is None] - if remaining_idx: - open_questions = ( - session.query(Question) - .filter(Question.status == QUESTION_STATUS_OPEN, scope) - .order_by(Question.first_asked_at.desc()) - .limit(OPEN_QUESTION_LIMIT) - .all() - ) - - if open_questions: - llm_matches = await self._llm_match( - [proposed[i].text for i in remaining_idx], - open_questions, - ) - for slot, original_idx in enumerate(remaining_idx): - matched_id = llm_matches.get(slot) - if matched_id is not None: - matched = next((q for q in open_questions if q.id == matched_id), None) - if matched and self._binding_allowed( - proposed[original_idx].text, matched, standing_ids - ): - resolved[original_idx] = matched - - previous_link: dict[int, int] = {} - still_unmatched = [i for i, r in enumerate(resolved) if r is None] - for i in still_unmatched: - norm = normalized[i] - if not norm: - continue - prior = ( - session.query(Question.id) - .filter( - Question.status == QUESTION_STATUS_RESOLVED, - Question.normalized_text == norm, - scope, - ) - .order_by(Question.resolved_at.desc()) - .first() - ) - if prior: - previous_link[i] = prior[0] - - now = utcnow() - for i, q in enumerate(resolved): - if q is not None: - continue - new_q = Question( - text=proposed[i].text, - normalized_text=normalized[i], - first_asked_at=now, - status=QUESTION_STATUS_OPEN, - is_primary=proposed[i].is_primary, - previous_question_id=previous_link.get(i), - ) - session.add(new_q) - resolved[i] = new_q - - session.flush() - return resolved # type: ignore[return-value] - - @staticmethod - def _binding_allowed(proposed_text: str, matched: Question, standing_ids: set[int]) -> bool: - """ - Whether an LLM-proposed match may stand. - - Always yes for an emergent question -- that path is unchanged. For a - *declared* standing question the LLM's answer additionally has to clear - :data:`STANDING_BINDING_MIN_OVERLAP`, and a refused bind is not a - failure: the proposed question simply becomes a new emergent Question, - and the standing question is reported as unmoved, which is the honest - answer when the coverage was about something else. - - Note the exact-normalized-text path above needs no gate: identical text - is already the tightest match there is. - """ - if matched.id not in standing_ids: - return True - - overlap = binding_overlap(proposed_text, matched.text) - if overlap >= STANDING_BINDING_MIN_OVERLAP: - return True - - logger.info( - f"Refused to bind {proposed_text!r} to standing question Q{matched.id}: " - f"subject overlap {overlap:.2f} < {STANDING_BINDING_MIN_OVERLAP}" - ) - return False - - async def _llm_match( - self, proposed_texts: list[str], open_questions: list[Question] - ) -> dict[int, int | None]: - """One Haiku call: maps each proposed slot index to a matched_id or None.""" - proposed_block = "\n".join(f"[{i}] {text}" for i, text in enumerate(proposed_texts)) - open_block = "\n".join( - f"[{q.id}] (asked {q.first_asked_at.date().isoformat()}) {q.text}" - for q in open_questions - ) - prompt = QUESTION_MATCHING_PROMPT.format( - proposed_block=proposed_block, - open_block=open_block, - ) - - try: - raw = await self.client.analyze( - system_prompt="You match unresolved questions across runs. Be conservative.", - user_message=prompt, - effort="low", - max_tokens=MATCHER_MAX_TOKENS, - ) - except Exception as e: - logger.warning(f"Question matcher LLM call failed; treating all as new: {e}") - return {} - - parsed = parse_claude_json(raw, label="question matcher response") - results: dict[int, int | None] = {} - for entry in parsed.get("matches", []): - idx = entry.get("proposed_index") - matched_id = entry.get("matched_id") - if isinstance(idx, int): - results[idx] = matched_id if isinstance(matched_id, int) else None - return results diff --git a/src/context/standing_agenda.py b/src/context/standing_agenda.py deleted file mode 100644 index e0ab016..0000000 --- a/src/context/standing_agenda.py +++ /dev/null @@ -1,260 +0,0 @@ -""" -Standing questions: the agenda a beat declares, rather than the one its -coverage happens to raise. - -Emergent Questions are discovered -- the graph notices what coverage leaves -unresolved. A **standing question** is the other direction: the human writes it -into ``config/beats/.json`` and the beat carries it whether or not this -week's coverage touches it. That inversion is the whole point, so the -load-bearing behaviour here is the *quiet* case: a standing question with no -coverage this run is reported as unmoved, never filtered out as an empty -section. "No movement on CMMC Phase 2 this week" is a finding. - -Two things follow from that: - -* **Seeding is idempotent and beat-bound.** A declared question becomes a real - ``Question`` row on the beat's first run, joined to the beat through - ``beat_standing_questions``. Re-running seeds nothing new. The join is what - keeps a never-yet-moved question inside the beat's scope instead of leaking - into the person brief's ledger -- see ``beat_scope.question_scope_filter``. -* **The agenda is computed at store time, not at render time.** It lands in - ``synthesis_data["metadata"]["standing_agenda"]``, so ``--from-run`` replays - the same agenda the live run reported, and no renderer needs database access. - -Added 2026-08-26 for backlog task 007. -""" - -from __future__ import annotations - -import logging -from typing import Any - -from sqlalchemy.orm import Session - -from ..database.models import ( - PREDICTION_STATUS_OPEN, - QUESTION_STATUS_OPEN, - BeatStandingQuestion, - Prediction, - Question, - QuestionSituation, -) -from ..utils import utcnow -from .beat_scope import beat_synthesis_ids, scoped_appearance_count -from .question_matcher import normalize_question - -logger = logging.getLogger(__name__) - - -def seed_standing_questions( - session: Session, beat_id: int, declared: tuple[str, ...] | list[str] -) -> list[Question]: - """ - Make sure every question this beat declares exists in the graph. - - Returns the ``Question`` rows for the declared agenda, in declaration - order. The caller commits. - - Idempotent by normalized text within the beat. An existing open Question in - this beat's scope with the same normalized text is *adopted* rather than - duplicated: when a beat has been raising a question emergently and the - human then declares it, that is the same question and its accumulated - history should survive the declaration. - - Nothing is ever retired here. Removing a question from the config stops it - being declared; it does not close the Question, because closing a question - is a judgement and this function makes none. - """ - if not declared: - return [] - - existing_rows = { - row.normalized_text: row - for row in session.query(BeatStandingQuestion).filter( - BeatStandingQuestion.beat_id == beat_id - ) - } - - seeded: list[Question] = [] - now = utcnow() - for text in declared: - normalized = normalize_question(text) - if not normalized: - # The config loader rejects blank declarations, so this can only be - # punctuation-only text. Skipping it silently would drop an agenda - # item, so say so. - logger.warning(f"Standing question {text!r} normalizes to nothing; not seeded") - continue - - link = existing_rows.get(normalized) - if link is not None: - question = session.query(Question).filter(Question.id == link.question_id).first() - if question is not None: - seeded.append(question) - continue - # The declaration outlived its Question row. Re-seed rather than - # carry a dangling agenda item. - session.delete(link) - session.flush() - - question = _adopt_or_create(session, beat_id, text, normalized, now) - session.add( - BeatStandingQuestion( - beat_id=beat_id, - question_id=question.id, - declared_text=text, - normalized_text=normalized, - declared_at=now, - ) - ) - session.flush() - seeded.append(question) - logger.info(f"Seeded standing question Q{question.id} for beat {beat_id}: {text!r}") - - return seeded - - -def _adopt_or_create( - session: Session, beat_id: int, text: str, normalized: str, now: Any -) -> Question: - """Reuse this beat's open Question with the same normalized text, or make one.""" - already_this_beat = ( - session.query(Question) - .join(QuestionSituation, QuestionSituation.question_id == Question.id) - .filter( - Question.normalized_text == normalized, - Question.status == QUESTION_STATUS_OPEN, - QuestionSituation.synthesis_id.in_(beat_synthesis_ids(beat_id)), - ) - .first() - ) - if already_this_beat is not None: - logger.info( - f"Standing question {text!r} adopts existing Q{already_this_beat.id} " - f"already raised by beat {beat_id}" - ) - return already_this_beat - - question = Question( - text=text, - normalized_text=normalized, - first_asked_at=now, - status=QUESTION_STATUS_OPEN, - is_primary=True, - ) - session.add(question) - session.flush() - return question - - -def build_standing_agenda( - session: Session, - beat_id: int, - situations: list[dict[str, Any]], - movement: dict[int, list[int]], -) -> list[dict[str, Any]]: - """ - Report this run's movement against every declared standing question. - - ``movement`` maps a Question id to the zero-based indices of the situations - that bound to it in this run. A standing question absent from that mapping - did not move, and it still gets an entry -- with ``moved`` false and the - date it last moved, or ``None`` if it never has. Filtering those out is the - one thing this function must never do. - - Returns entries in declaration order. Renderers consume this verbatim; it - contains no styling and no judgement about whether a question is well-posed. - """ - links = ( - session.query(BeatStandingQuestion) - .filter(BeatStandingQuestion.beat_id == beat_id) - .order_by(BeatStandingQuestion.declared_at.asc(), BeatStandingQuestion.id.asc()) - .all() - ) - - agenda: list[dict[str, Any]] = [] - for link in links: - question = session.query(Question).filter(Question.id == link.question_id).first() - if question is None: - continue - - moved_indices = sorted(movement.get(int(question.id), [])) - entry: dict[str, Any] = { - "question_id": int(question.id), - "text": question.text, - "status": question.status, - "declared_at": _date(link.declared_at), - "moved": bool(moved_indices), - "moved_in": [ - { - "situation_index": index + 1, - "title": _situation_title(situations, index), - } - for index in moved_indices - ], - # Counted within this beat's scope, and inclusive of this run: the - # number the reader sees is "how many runs of this beat have moved - # it", not "how many times anything anywhere raised it". - "appearance_count": scoped_appearance_count(session, question.id, beat_id) - + len(moved_indices), - "last_moved_at": _last_moved_at(session, beat_id, int(question.id)), - "watching": _open_observables(session, int(question.id)), - } - if link.declared_text and link.declared_text != question.text: - entry["declared_text"] = link.declared_text - agenda.append(entry) - - return agenda - - -def _situation_title(situations: list[dict[str, Any]], index: int) -> str: - if 0 <= index < len(situations): - title = situations[index].get("title") - if isinstance(title, str): - return title - return "" - - -def _last_moved_at(session: Session, beat_id: int, question_id: int) -> str | None: - """The date this question last appeared in one of *this beat's* syntheses.""" - row = ( - session.query(QuestionSituation.observed_at) - .filter( - QuestionSituation.question_id == question_id, - QuestionSituation.synthesis_id.in_(beat_synthesis_ids(beat_id)), - ) - .order_by(QuestionSituation.observed_at.desc()) - .first() - ) - return _date(row[0]) if row and row[0] else None - - -def _open_observables(session: Session, question_id: int) -> list[str]: - """ - Open predictions keyed to this question -- what would have to be observed - to change the picture. Read-only: the ledger is graded elsewhere, and - nothing here creates, resolves or expires a Prediction. - """ - rows = ( - session.query(Prediction) - .filter( - Prediction.question_id == question_id, - Prediction.status == PREDICTION_STATUS_OPEN, - ) - .order_by(Prediction.made_at.desc()) - .all() - ) - items: list[str] = [] - for row in rows: - observable = (row.observable_text or "").strip() - trigger = (row.trigger_condition or "").strip() - if observable and trigger: - items.append(f"{observable} -- {trigger}") - elif observable: - items.append(observable) - return items - - -def _date(value: Any) -> str | None: - """A datetime as ``YYYY-MM-DD``, or None.""" - return value.date().isoformat() if value is not None else None diff --git a/src/context/synthesizer.py b/src/context/synthesizer.py deleted file mode 100644 index 9b89828..0000000 --- a/src/context/synthesizer.py +++ /dev/null @@ -1,915 +0,0 @@ -""" -Two-Pass Narrative Synthesizer - -Pass 1: Cluster articles into topic groups. -Pass 2: For clusters with 3+ articles, produce examined situation narratives. - For clusters with 1-2 articles, produce thin coverage summaries. -""" - -import json -import logging -from typing import Any - -from ..config.beats import BeatConfig -from ..database.connection import get_db -from ..database.models import ( - BEAT_RUN_STATUS_COMPLETED, - AnalysisRun, - Article, - BeatRun, - ContextSnapshot, - DecisionEvidence, - NarrativeSynthesis, - Prediction, - Question, - QuestionSituation, -) -from ..prompts import load_analysis_rules -from ..prompts.synthesis import ( - CLUSTERING_PROMPT, - SITUATION_SYNTHESIS_PROMPT, - THIN_COVERAGE_PROMPT, -) -from ..utils import utcnow -from ..utils.profiler import profile -from ._json import parse_claude_json -from .beat_scope import ensure_beat, scoped_appearance_count -from .claude_client import ClaudeClient -from .cross_cluster_reconciler import CrossClusterReconciler -from .curator import ContextCurator -from .decision_router import DecisionRouter -from .entity_matcher import item_text -from .frame_manager import FrameManager -from .institutional_activity import observe_activity, record_mentions -from .prediction_tracker import PredictionTracker -from .question_matcher import ProposedQuestion, QuestionMatcher -from .standing_agenda import build_standing_agenda, seed_standing_questions - -logger = logging.getLogger(__name__) - -ANALYSIS_THRESHOLD = 2 # Minimum articles for full situation analysis - - -class NarrativeSynthesizer: - """Two-pass situation-based narrative synthesizer.""" - - def __init__( - self, - topic_filters: dict | None = None, - client: ClaudeClient | None = None, - beat: BeatConfig | None = None, - ): - """ - Args: - topic_filters: Optional curation filters. - client: Optional Claude client. When omitted (the production path), - every collaborator builds its own client with its own model, as - before. When supplied, it is injected into all of them, which is - the seam tests use to run without an ANTHROPIC_API_KEY. - beat: Optional beat. Scopes both the articles curated and the slice - of the commitment graph this run reads and writes. ``None`` is - the default person-profile brief, unchanged. - """ - self.topic_filters = topic_filters or {} - self.beat = beat - # Set on the first store/check of a beat run; None for the default path. - self.beat_id: int | None = None - # This run's institutional mention counts, keyed by beat_entities.id. - # Populated by the (deterministic, model-free) coverage pass and drained - # into entity_mentions inside the synthesis transaction. - self.entity_counts: dict[int, int] = {} - self.entity_items_scanned: int = 0 - self.curator = ContextCurator(topic_filters=self.topic_filters, beat=beat) - self.client = client or ClaudeClient() - self.analysis_rules = load_analysis_rules() - self.frame_manager = FrameManager(self.client) - self.question_matcher = QuestionMatcher(client) - self.prediction_tracker = PredictionTracker(client) - self.decision_router = DecisionRouter(client) - self.cross_cluster_reconciler = CrossClusterReconciler(client) - - async def synthesize(self, hours: int = 48, max_articles: int = 50) -> dict[str, Any]: - """ - Two-pass synthesis: cluster articles, then analyze each situation. - - Returns: - { - "status": "success" | "no_articles" | "error", - "articles_analyzed": int, - "synthesis_id": int | None, - "synthesis_data": { - "situations": [...], - "thin_coverage": [...], - "metadata": {...} - } - } - """ - with profile("SYNTHESIS_TOTAL"): - logger.info(f"Starting two-pass synthesis for last {hours} hours") - - # Curate context - context = await self.curator.curate_for_narrative_synthesis(hours, max_articles) - articles = context.get("articles", []) - - if not articles: - logger.warning("No articles available for synthesis") - return {"articles_analyzed": 0, "synthesis_id": None, "status": "no_articles"} - - try: - # Register the beat before anything touches the graph. Every - # scoped read and write below keys off this id, so failing to - # get one must abort the run rather than quietly dump the - # beat's questions into the default scope. - if self.beat is not None: - self.beat_id = self._register_beat(self.beat) - - # Coverage pass: count which of the beat's declared - # institutions this run's items mention, and read those counts - # against each entity's trailing average. Deterministic word - # boundary alias matching -- no model call, no token cost, and - # the same articles always produce the same counts. - institutional_activity = self._observe_institutional_activity(articles) - - # Pre-pass: grade the open-prediction ledger against today's - # coverage before any new analysis. This keeps the tool's own - # forward-looking statements auditable. - with profile("PREDICTION_CHECK"): - prediction_check = await self._check_predictions(articles) - - # Pass 1: Cluster articles - with profile("PASS_1_CLUSTERING"): - clusters = await self._cluster_articles(articles) - - if not clusters: - logger.error("Clustering returned no clusters") - return { - "articles_analyzed": len(articles), - "synthesis_id": None, - "status": "error", - "error": "Article clustering failed", - } - - logger.info( - f"Pass 1 complete: {len(clusters)} clusters from {len(articles)} articles" - ) - - # Split clusters by coverage threshold - full_clusters = [c for c in clusters if len(c["article_ids"]) >= ANALYSIS_THRESHOLD] - thin_clusters = [c for c in clusters if len(c["article_ids"]) < ANALYSIS_THRESHOLD] - - logger.info( - f"Full analysis: {len(full_clusters)} clusters | " - f"Thin coverage: {len(thin_clusters)} clusters" - ) - - # Build citation map from all articles - citation_map = self._build_citation_map(articles) - - # Pass 2a: Full situation analysis for clusters with 3+ articles - situations = [] - with profile("PASS_2_SITUATIONS"): - for cluster in full_clusters: - cluster_articles = [ - a - for a in articles - if a.get("id") in cluster["article_ids"] - or articles.index(a) + 1 in cluster["article_ids"] - ] - if not cluster_articles: - cluster_articles = [ - articles[i - 1] - for i in cluster["article_ids"] - if 0 < i <= len(articles) - ] - - if not cluster_articles: - continue - - # Check for known frames - frame_prompt_addition = "" - existing_cluster = self.frame_manager.find_matching_cluster(cluster) - - if existing_cluster: - validated_frames = self.frame_manager.get_validated_frames( - existing_cluster.id - ) - if validated_frames: - frame_prompt_addition = self.frame_manager.build_frame_aware_prompt( - existing_cluster.name, validated_frames - ) - logger.info( - f"Injecting {len(validated_frames)} known frames " - f"for '{cluster['title']}'" - ) - - # Run situation synthesis - situation = await self._analyze_situation( - cluster_articles, - cluster, - citation_map, - frame_prompt_addition=frame_prompt_addition, - ) - - if not situation: - continue - - situations.append(situation) - - # Frame discovery: if no known frames, discover them, - # then tag this cluster's articles against the result. - if not existing_cluster: - try: - discovery = await self.frame_manager.discover_frames( - cluster_articles, cluster - ) - if discovery: - tc_id = self.frame_manager.store_discovered_frames( - cluster, discovery - ) - if tc_id: - frames = self.frame_manager.get_cluster_frames(tc_id) - await self.frame_manager.classify_articles_to_frames( - cluster_articles, frames - ) - except Exception as e: - logger.warning( - f"Frame discovery failed for '{cluster['title']}': {e}" - ) - - # Known frames: tag articles and update gap tracking. - elif existing_cluster: - try: - cluster_frames = self.frame_manager.get_cluster_frames( - existing_cluster.id - ) - if cluster_frames: - await self.frame_manager.classify_articles_to_frames( - cluster_articles, cluster_frames - ) - except Exception as e: - logger.warning( - f"Frame classification failed for '{cluster['title']}': {e}" - ) - - if situation.get("coverage_frame"): - absent = situation.get("coverage_frame", {}).get( - "de_emphasized", "" - ) - if absent: - self.frame_manager.update_frame_gaps( - existing_cluster.id, [absent] - ) - - # Pass 2b: Thin coverage summaries - thin_coverage = [] - if thin_clusters: - with profile("PASS_2_THIN"): - thin_coverage = await self._summarize_thin_clusters(thin_clusters, articles) - - # Pass 3: look across situations for meta-fractures (one - # underlying frame conflict surfacing in multiple topical - # guises). Empty result is the common case; we skip silently. - meta_fractures = [] - if situations: - with profile("PASS_3_RECONCILE"): - try: - meta_fractures = await self.cross_cluster_reconciler.reconcile( - situations - ) - except Exception as e: - logger.warning(f"Cross-cluster reconciliation failed: {e}") - - # Assemble output - synthesis_data = { - "situations": situations, - "thin_coverage": thin_coverage, - "meta_fractures": meta_fractures, - "metadata": { - "articles_analyzed": len(articles), - "clusters_total": len(clusters), - "clusters_analyzed": len(full_clusters), - "clusters_thin": len(thin_clusters), - "analysis_threshold": f"{ANALYSIS_THRESHOLD}+ articles", - "generated_at": utcnow().isoformat(), - "citation_map": citation_map, - "prediction_check": prediction_check, - }, - } - if institutional_activity is not None: - synthesis_data["metadata"]["institutional_activity"] = institutional_activity - - # Store in database (also resolves Questions and writes joins) - synthesis_id = await self._store_synthesis( - synthesis_data=synthesis_data, - articles_count=len(articles), - context=context, - ) - - logger.info( - f"Synthesis complete: {synthesis_id} " - f"({len(situations)} situations, {len(thin_coverage)} thin)" - ) - - return { - "status": "success", - "articles_analyzed": len(articles), - "synthesis_id": synthesis_id, - "synthesis_data": synthesis_data, - } - - except Exception as e: - logger.error(f"Synthesis failed: {e}", exc_info=True) - return { - "articles_analyzed": len(articles), - "synthesis_id": None, - "status": "error", - "error": str(e), - } - - # ========================================================================= - # Pass 1: Clustering - # ========================================================================= - - async def _cluster_articles(self, articles: list[dict]) -> list[dict]: - """ - Group articles into topic clusters using a lightweight Claude call. - - Returns list of {"title": str, "article_ids": [int], "keywords": [str]} - """ - # Build article summaries for clustering (titles + first paragraph only) - article_summaries = [] - for i, article in enumerate(articles, 1): - title = article.get("title", "Untitled") - content = article.get("content", article.get("description", "")) - # First 200 chars of content for clustering - snippet = content[:200].strip() if content else "" - article_summaries.append(f"[{i}] {title}\n {snippet}") - - articles_text = "\n\n".join(article_summaries) - prompt = CLUSTERING_PROMPT.format(articles=articles_text) - - response = await self.client.analyze( - system_prompt="You are a news article classifier. Group articles by topic.", - user_message=prompt, - effort="low", - ) - - parsed = parse_claude_json(response, label="clustering response") - return parsed.get("clusters", []) - - # ========================================================================= - # Pass 2a: Full situation analysis - # ========================================================================= - - async def _analyze_situation( - self, - cluster_articles: list[dict], - cluster: dict, - _citation_map: dict, - frame_prompt_addition: str = "", - ) -> dict | None: - """ - Produce an examined narrative for a single situation cluster. - - Args: - frame_prompt_addition: Optional FRAME_AWARE_SYNTHESIS_PROMPT to append. - - Returns the situation dict or None on failure. - """ - # Build article reference list for this cluster - article_refs = [] - for i, article in enumerate(cluster_articles, 1): - title = article.get("title", "Untitled") - source = article.get("source", "Unknown") - date = article.get("published_date", "No date") - article_refs.append(f"[{i}] {title} - {source} ({date})") - - article_ref_list = "\n".join(article_refs) - - prompt = SITUATION_SYNTHESIS_PROMPT.format( - analysis_rules=self.analysis_rules, - article_ref_list=article_ref_list, - ) - - # Inject frame-aware prompt if known frames exist - if frame_prompt_addition: - prompt = f"{prompt}\n\n{frame_prompt_addition}" - - # Build context with full article content for this cluster - context = { - "user_profile": self.curator._format_user_profile(), - "articles": cluster_articles, - "instructions": self.curator._get_synthesis_instructions(), - } - - response = await self.client.analyze_with_context( - context=context, - task=prompt, - effort="high", - ) - - situation = parse_claude_json(response, label="situation analysis response") - - # Validate minimal structure - if "title" not in situation and "narrative" not in situation: - logger.warning(f"Situation analysis returned invalid structure for: {cluster['title']}") - return None - - return situation - - # ========================================================================= - # Pass 2b: Thin coverage summaries - # ========================================================================= - - async def _summarize_thin_clusters( - self, thin_clusters: list[dict], articles: list[dict] - ) -> list[dict]: - """Produce one-line summaries for clusters with insufficient coverage.""" - cluster_descriptions = [] - for cluster in thin_clusters: - # Gather source names for this cluster - sources = [] - for aid in cluster["article_ids"]: - idx = aid - 1 # 1-indexed to 0-indexed - if 0 <= idx < len(articles): - sources.append(articles[idx].get("source", "Unknown")) - - cluster_descriptions.append( - f"- {cluster['title']} ({len(cluster['article_ids'])} articles, " - f"sources: {', '.join(set(sources))})" - ) - - clusters_text = "\n".join(cluster_descriptions) - prompt = THIN_COVERAGE_PROMPT.format(clusters=clusters_text) - - response = await self.client.analyze( - system_prompt="You are summarizing news topics with thin coverage.", - user_message=prompt, - effort="low", - ) - - parsed = parse_claude_json(response, label="thin coverage response") - return parsed.get("thin_coverage", []) - - # ========================================================================= - # Utilities - # ========================================================================= - - @staticmethod - def _build_citation_map(articles: list[dict]) -> dict: - """Build authoritative citation map from articles.""" - citation_map = {} - for i, article in enumerate(articles, 1): - citation_map[str(i)] = { - "article_id": article.get("id"), - "title": article.get("title", "Untitled"), - "source": article.get("source", "Unknown"), - "url": article.get("url", ""), - } - return citation_map - - async def _store_synthesis( - self, - synthesis_data: dict[str, Any], - articles_count: int, - context: dict[str, Any] | None = None, - ) -> int | None: - """Store synthesis, resolve Questions, and write join rows in one txn.""" - try: - with get_db() as session: - run = AnalysisRun( - run_type="situation_synthesis", - status="completed", - started_at=utcnow(), - completed_at=utcnow(), - articles_processed=articles_count, - context_token_count=self._estimate_tokens(context) if context else None, - # Read from the client rather than hardcoded. Changed 2026-08-26: - # this field is provenance -- it records which model produced the - # synthesis -- and a literal here records whatever was true when the - # line was written, not what actually ran. It had said - # claude-sonnet-4-20250514 since that model was retired. - claude_model=self.client.model, - ) - session.add(run) - session.flush() - - context_snapshot_id = None - if context: - article_ids = [a.get("id") for a in context.get("articles", []) if "id" in a] - snapshot = ContextSnapshot( - synthesis_id=None, - article_ids=str(article_ids), - context_size_tokens=self._estimate_tokens(context), - user_profile_hash=self._hash_profile(context.get("user_profile")), - historical_summaries=context.get("memory", ""), - instructions=context.get("instructions", ""), - ) - session.add(snapshot) - session.flush() - context_snapshot_id = snapshot.id - - # Resolve Questions against persistent graph, then enrich - # synthesis_data with question identity metadata so the - # formatter can render Q-id prefixes without DB access. - situations = synthesis_data.get("situations", []) - question_plan = self._collect_proposed_questions(situations) - if question_plan: - proposed = [pq for _, _, pq in question_plan] - resolved = await self.question_matcher.resolve_questions( - proposed, session, beat_id=self.beat_id - ) - self._enrich_situations_with_questions( - situations, question_plan, resolved, session, self.beat_id - ) - else: - resolved = [] - - # Report movement against this beat's declared agenda. Computed - # here, before the synthesis row is built, so it is stored in - # the payload and replays identically under --from-run. Every - # declared question gets an entry whether or not it moved -- - # "nothing moved on this" is the finding, not an empty section. - if self.beat_id is not None: - synthesis_data.setdefault("metadata", {})["standing_agenda"] = ( - build_standing_agenda( - session, - self.beat_id, - situations, - self._question_movement(question_plan, resolved), - ) - ) - - # Route situation evidence into open decision factors. Done - # before the synthesis row write so the routing summary lands - # in the stored synthesis_data blob. - routed_evidence = await self.decision_router.route_evidence(situations, session) - synthesis_data.setdefault("metadata", {})["decision_routing"] = ( - self._build_decision_summary(routed_evidence, session) - ) - - exec_summary = ( - situations[0].get("title", "No summary available") - if situations - else "No situations identified" - ) - - synthesis = NarrativeSynthesis( - analysis_run_id=run.id, - context_snapshot_id=context_snapshot_id, - user_profile_version="1.0", - synthesis_data=synthesis_data, - executive_summary=exec_summary, - articles_analyzed=articles_count, - generated_at=utcnow(), - ) - session.add(synthesis) - session.flush() - - if context_snapshot_id: - session.query(ContextSnapshot).filter_by(id=context_snapshot_id).update( - {"synthesis_id": synthesis.id} - ) - - # Write QuestionSituation join rows now that we have synthesis.id. - primary_question_by_situation: dict[int, Question] = {} - for (sit_idx, slot, _pq), q in zip(question_plan, resolved, strict=True): - session.add( - QuestionSituation( - question_id=q.id, - synthesis_id=synthesis.id, - situation_index=sit_idx, - ) - ) - if slot == "primary": - primary_question_by_situation[sit_idx] = q - - # Persist this run's what_to_watch observables as Predictions, - # each keyed to its situation's primary Question. - for sit_idx, observable, trigger in self._collect_predictions(situations): - question = primary_question_by_situation.get(sit_idx) - if question is None: - continue - session.add( - Prediction( - question_id=question.id, - observable_text=observable, - trigger_condition=trigger, - made_in_synthesis_id=synthesis.id, - ) - ) - - # Persist routed decision evidence, attaching synthesis_id and - # the situation's primary Question. - for routed in routed_evidence: - primary_q = primary_question_by_situation.get(routed.situation_index) - session.add( - DecisionEvidence( - decision_id=routed.decision_id, - factor_id=routed.factor_id, - synthesis_id=synthesis.id, - question_id=primary_q.id if primary_q else None, - situation_excerpt=routed.excerpt, - direction=routed.direction, - epistemic_status=routed.epistemic_status, - ) - ) - - if context: - article_ids = [ - int(a.get("id")) for a in context.get("articles", []) if a.get("id") - ] - if article_ids: - session.query(Article).filter(Article.id.in_(article_ids)).update( - {"last_included_in_synthesis": utcnow()}, - synchronize_session=False, - ) - - # Attribute this run to its beat. Written in the same - # transaction as the synthesis, so a stored synthesis is either - # attributed or does not exist -- there is no window in which a - # beat's questions look like the default brief's. - if self.beat_id is not None: - beat_run = BeatRun( - beat_id=self.beat_id, - analysis_run_id=run.id, - synthesis_id=synthesis.id, - status=BEAT_RUN_STATUS_COMPLETED, - started_at=run.started_at, - completed_at=utcnow(), - articles_analyzed=articles_count, - feeds_resolved=len(self.curator.beat_feed_urls), - ) - session.add(beat_run) - - # This run's institutional counts, written in the same - # transaction as the run they describe. Every declared - # entity gets a row, zeroes included: a quiet office is an - # observation, and without the zero the trailing average - # would only ever average the busy days. - if self.entity_counts: - session.flush() - record_mentions( - session, - self.entity_counts, - beat_run_id=beat_run.id, - synthesis_id=synthesis.id, - items_scanned=self.entity_items_scanned, - ) - - session.commit() - logger.info(f"Stored synthesis: {synthesis.id}") - return synthesis.id - - except Exception as e: - logger.error(f"Failed to store synthesis: {e}") - return None - - @staticmethod - def _register_beat(beat: BeatConfig) -> int: - """ - Get or create this beat's ``beats`` row and return its id, seeding the - standing questions it declares in the same transaction. - - Deliberately not caught: without a beat id the run cannot be scoped, - and an unscoped beat run would corrupt the default brief's graph. The - seeding is here, before any coverage is read, because a declared - question exists because a human declared it -- not because an article - mentioned it. - """ - with get_db() as session: - beat_id = ensure_beat(session, beat) - seed_standing_questions(session, beat_id, beat.standing_questions) - session.commit() - return beat_id - - @staticmethod - def _question_movement( - plan: list[tuple[int, str, ProposedQuestion]], resolved: list[Question] - ) -> dict[int, list[int]]: - """ - Map each Question id this run bound to the situations that bound to it. - - Keys are Question ids, values are zero-based situation indices. This is - the same binding the QuestionSituation rows record; it is read off the - plan rather than the database because the join rows need a synthesis id - that does not exist yet at this point in the transaction. - """ - movement: dict[int, set[int]] = {} - for (sit_idx, _slot, _pq), question in zip(plan, resolved, strict=True): - # A set, not a list: one question can fill both the primary and a - # secondary slot of the same situation, and that is one movement. - movement.setdefault(int(question.id), set()).add(sit_idx) - return {question_id: sorted(indices) for question_id, indices in movement.items()} - - def _observe_institutional_activity(self, articles: list[dict]) -> dict | None: - """ - Count this run's mentions of the beat's declared coverage entities. - - Returns the brief-facing payload, or ``None`` when there is nothing to - observe -- the default person brief, a beat with an empty ``coverage`` - block, or a database that predates the tables. A failure here is - logged and swallowed: institutional activity is an additive reading of - articles the run already has, and losing a brief over it would be a - worse outcome than losing one section of it. - - No Claude call happens on this path, by design. If alias matching ever - proves insufficient, that is a reason to say so, not to reach for a - model: a count that a model produced is not reproducible from the same - articles tomorrow, and an unreproducible baseline cannot support a - delta. - """ - if self.beat is None or self.beat_id is None or not self.beat.entities: - return None - - try: - texts = [item_text(article) for article in articles] - with get_db() as session: - observation = observe_activity(session, self.beat_id, self.beat.entities, texts) - session.commit() - self.entity_counts = observation.counts_by_entity_id - self.entity_items_scanned = observation.items_scanned - logger.info( - f"Institutional activity: {len(observation.entities)} entities with " - f"history or mentions, {observation.never_observed} never observed" - ) - return observation.as_dict() - except Exception as e: - logger.warning(f"Institutional activity pass failed; continuing without it: {e}") - self.entity_counts = {} - return None - - async def _check_predictions(self, articles: list[dict]) -> dict: - """Grade the open-prediction ledger against today's coverage.""" - try: - with get_db() as session: - return await self.prediction_tracker.check_open_predictions( - articles, session, beat_id=self.beat_id - ) - except Exception as e: - logger.warning(f"Prediction check failed; continuing without it: {e}") - return { - "checked": 0, - "triggered": 0, - "contradicted": 0, - "expired": 0, - "still_open": 0, - } - - @staticmethod - def _build_decision_summary(routed_evidence, session) -> list[dict]: - """ - Group routed evidence by decision into a compact brief summary: - ``[{"decision": name, "factors": [{"name", "direction"}]}]``. - """ - from ..database.models import Decision, DecisionFactor - - if not routed_evidence: - return [] - - decision_ids = {r.decision_id for r in routed_evidence} - factor_ids = {r.factor_id for r in routed_evidence} - decisions = { - d.id: d.name - for d in session.query(Decision).filter(Decision.id.in_(decision_ids)).all() - } - factors = { - f.id: f.name - for f in session.query(DecisionFactor).filter(DecisionFactor.id.in_(factor_ids)).all() - } - - grouped: dict[int, list[dict]] = {} - for r in routed_evidence: - grouped.setdefault(r.decision_id, []).append( - {"name": factors.get(r.factor_id, "(factor)"), "direction": r.direction} - ) - return [ - {"decision": decisions.get(did, "(decision)"), "factors": factor_list} - for did, factor_list in grouped.items() - ] - - @staticmethod - def _collect_predictions(situations: list[dict]) -> list[tuple[int, str, str]]: - """ - Walk situations and extract what_to_watch observables. - - Returns ``[(situation_index, observable_text, trigger_condition), ...]``. - """ - out: list[tuple[int, str, str]] = [] - for s_idx, situation in enumerate(situations): - futures = situation.get("where_this_goes") or {} - watch = futures.get("what_to_watch") - if isinstance(watch, list): - for entry in watch: - if not isinstance(entry, dict): - continue - observable = (entry.get("observable") or "").strip() - trigger = (entry.get("trigger_condition") or "").strip() - if observable and trigger: - out.append((s_idx, observable, trigger)) - return out - - @staticmethod - def _collect_proposed_questions( - situations: list[dict], - ) -> list[tuple[int, str, ProposedQuestion]]: - """ - Walk situations and extract their unresolved questions. - - Returns ``[(situation_index, slot, ProposedQuestion), ...]`` where - ``slot`` is "primary" or "secondary:N". Order matches the order - situations + secondaries are emitted, so callers can zip in parallel. - """ - plan: list[tuple[int, str, ProposedQuestion]] = [] - for s_idx, situation in enumerate(situations): - futures = situation.get("where_this_goes") or {} - uq = futures.get("unresolved_questions") - if isinstance(uq, dict): - primary = uq.get("primary") - if isinstance(primary, str) and primary.strip(): - plan.append((s_idx, "primary", ProposedQuestion(primary.strip(), True))) - for i, sec in enumerate(uq.get("secondary") or []): - if isinstance(sec, str) and sec.strip(): - plan.append((s_idx, f"secondary:{i}", ProposedQuestion(sec.strip(), False))) - elif isinstance(uq, str) and uq.strip(): - # Backward compat for any synthesis emitting the old string form. - plan.append((s_idx, "primary", ProposedQuestion(uq.strip(), True))) - else: - legacy = futures.get("unresolved_question") - if isinstance(legacy, str) and legacy.strip(): - plan.append((s_idx, "primary", ProposedQuestion(legacy.strip(), True))) - return plan - - @staticmethod - def _enrich_situations_with_questions( - situations: list[dict], - plan: list[tuple[int, str, ProposedQuestion]], - resolved: list[Question], - session, - beat_id: int | None = None, - ) -> None: - """ - Attach question identity metadata onto each situation's questions. - - Appearance counts are counted within ``beat_id``'s scope, so the run - number a brief prints is "the Nth time *this* subject raised it". With - no beat runs on record the scope is the whole graph, which is the count - this produced before beats existed. - """ - # Group resolved by situation_index + slot for easy lookup. - by_slot: dict[tuple[int, str], Question] = {} - for (sit_idx, slot, _pq), q in zip(plan, resolved, strict=True): - by_slot[(sit_idx, slot)] = q - - for sit_idx, situation in enumerate(situations): - futures = situation.get("where_this_goes") - if not isinstance(futures, dict): - continue - primary_q = by_slot.get((sit_idx, "primary")) - if primary_q: - appearance_count = scoped_appearance_count(session, primary_q.id, beat_id) + 1 - futures["unresolved_questions"] = { - "primary": { - "text": primary_q.text, - "question_id": primary_q.id, - "first_asked_at": primary_q.first_asked_at.isoformat(), - "appearance_count": appearance_count, - "previous_question_id": primary_q.previous_question_id, - }, - "secondary": [], - } - # Drop the legacy key if it was present. - futures.pop("unresolved_question", None) - - sec_idx = 0 - while True: - sec_q = by_slot.get((sit_idx, f"secondary:{sec_idx}")) - if not sec_q: - break - appearance_count = scoped_appearance_count(session, sec_q.id, beat_id) + 1 - futures.setdefault("unresolved_questions", {"primary": None, "secondary": []}) - futures["unresolved_questions"].setdefault("secondary", []).append( - { - "text": sec_q.text, - "question_id": sec_q.id, - "first_asked_at": sec_q.first_asked_at.isoformat(), - "appearance_count": appearance_count, - "previous_question_id": sec_q.previous_question_id, - } - ) - sec_idx += 1 - - @staticmethod - def _estimate_tokens(context: dict[str, Any]) -> int: - """Rough token count estimate (1 token ~ 4 chars).""" - return len(json.dumps(context)) // 4 - - @staticmethod - def _hash_profile(profile: dict[str, Any] | None) -> str: - """Hash user profile for tracking.""" - import hashlib - - if not profile: - return "none" - profile_str = json.dumps(profile, sort_keys=True) - return hashlib.sha256(profile_str.encode()).hexdigest() diff --git a/src/context/topic_matcher.py b/src/context/topic_matcher.py deleted file mode 100644 index 0a9c347..0000000 --- a/src/context/topic_matcher.py +++ /dev/null @@ -1,611 +0,0 @@ -""" -Topic Matcher - Article filtering by topic and geographic scope -Uses keyword matching and metadata to filter articles without AI inference -""" - -import logging - -from src.database.models import Article -from src.utils.profile_loader import UserProfile - -logger = logging.getLogger(__name__) - -# Topic keyword dictionaries with broad interpretation -TOPIC_KEYWORDS = { - "cybersecurity": { - "core": [ - "cybersecurity", - "cyber security", - "infosec", - "information security", - "cyber", - "security breach", - "data security", - "network security", - ], - "threats": [ - "ransomware", - "malware", - "phishing", - "breach", - "hack", - "hacker", - "zero day", - "zero-day", - "vulnerability", - "exploit", - "attack", - "threat actor", - "backdoor", - "trojan", - "spyware", - "botnet", - "ddos", - "denial of service", - "sql injection", - "xss", - ], - "tech": [ - "firewall", - "encryption", - "authentication", - "access control", - "vpn", - "ids", - "intrusion detection", - "penetration test", - "security patch", - "security update", - "mfa", - "multi-factor", - ], - "policy": [ - "cyber policy", - "data privacy", - "surveillance", - "CISA", - "cybersecurity framework", - "nist", - "compliance", - "gdpr", - "hipaa", - "data protection", - "privacy law", - ], - "industry": [ - "supply chain security", - "critical infrastructure", - "industrial control", - "scada", - "ot security", - "third-party risk", - "vendor security", - ], - "geopolitical": [ - "cyber warfare", - "state-sponsored", - "espionage", - "nation-state", - "apt", - "advanced persistent threat", - "cyber attack", - "cyber espionage", - ], - "entities": [ - "crowdstrike", - "palo alto", - "mandiant", - "cisa", - "nsa", - "cisco", - "fortinet", - "microsoft security", - "google security", - "amazon security", - "cloudflare", - ], - }, - "ai/ml": { - "core": [ - "artificial intelligence", - "ai", - "machine learning", - "ml", - "deep learning", - "neural network", - "ai model", - "ai system", - ], - "tech": [ - "llm", - "large language model", - "gpt", - "transformer", - "generative ai", - "computer vision", - "nlp", - "natural language", - "reinforcement learning", - "supervised learning", - "unsupervised learning", - "training data", - "ai training", - "inference", - ], - "policy": [ - "ai regulation", - "ai safety", - "ai ethics", - "ai governance", - "ai bias", - "algorithmic fairness", - "responsible ai", - "ai transparency", - "explainable ai", - ], - "entities": [ - "openai", - "anthropic", - "google ai", - "microsoft ai", - "deepmind", - "meta ai", - "nvidia", - "hugging face", - ], - }, - "software development": { - "core": [ - "software development", - "programming", - "coding", - "developer", - "software engineer", - "devops", - "agile", - "scrum", - ], - "tech": [ - "git", - "github", - "ci/cd", - "docker", - "kubernetes", - "api", - "microservices", - "cloud native", - "serverless", - "react", - "python", - "javascript", - "java", - "go", - ], - "tools": [ - "ide", - "visual studio", - "vscode", - "intellij", - "jenkins", - "gitlab", - "jira", - "slack", - ], - }, - "cloud computing": { - "core": [ - "cloud computing", - "cloud", - "saas", - "paas", - "iaas", - "cloud services", - "cloud platform", - ], - "providers": [ - "aws", - "amazon web services", - "azure", - "microsoft azure", - "gcp", - "google cloud", - "ibm cloud", - "oracle cloud", - ], - "tech": [ - "s3", - "ec2", - "lambda", - "cloud storage", - "cloud database", - "cloud security", - "cloud migration", - "multi-cloud", - "hybrid cloud", - ], - }, - "technology policy": { - "core": [ - "technology policy", - "tech policy", - "tech regulation", - "digital policy", - "internet policy", - ], - "topics": [ - "net neutrality", - "data privacy", - "antitrust", - "section 230", - "content moderation", - "platform regulation", - "digital markets", - "competition policy", - "tech monopoly", - ], - "entities": ["ftc", "fcc", "doj", "eu commission", "congress"], - }, - "privacy": { - "core": [ - "privacy", - "data privacy", - "personal data", - "user data", - "privacy rights", - "data protection", - ], - "topics": [ - "gdpr", - "ccpa", - "surveillance", - "tracking", - "cookies", - "consent", - "data collection", - "privacy policy", - "encryption", - "anonymization", - ], - }, - "education": { - "core": [ - "education", - "school", - "university", - "college", - "student", - "teacher", - "classroom", - "learning", - ], - "topics": [ - "education policy", - "curriculum", - "standardized test", - "school funding", - "education budget", - "school board", - "online learning", - "edtech", - "student loan", - ], - }, - "transportation": { - "core": [ - "transportation", - "transit", - "commute", - "traffic", - "infrastructure", - "public transport", - ], - "topics": [ - "metro", - "subway", - "bus", - "train", - "highway", - "road", - "ev", - "electric vehicle", - "autonomous vehicle", - "self-driving", - ], - }, - "housing": { - "core": [ - "housing", - "real estate", - "property", - "home", - "apartment", - "rent", - "mortgage", - "housing market", - ], - "topics": [ - "affordable housing", - "housing crisis", - "zoning", - "development", - "construction", - "homeowner", - "renter", - "eviction", - "housing policy", - ], - }, -} - - -class TopicMatcher: - """ - Matches articles to topics using metadata and keyword expansion - Provides both topic filtering and geographic scope filtering - """ - - def __init__(self): - self.topic_keywords = TOPIC_KEYWORDS - - def matches_topic(self, article: Article, topic: str) -> tuple[bool, float]: - """ - Check if article matches topic using metadata and weighted scoring - - Args: - article: Article to check - topic: Topic name (e.g., 'cybersecurity', 'ai/ml') - - Returns: - (matches: bool, score: float) - Whether article matches and confidence score - """ - score = 0.0 - - # Get keywords for this topic - keywords = self.topic_keywords.get(topic, {}) - if not keywords: - logger.warning(f"Unknown topic: {topic}") - return False, 0.0 - - # Combine searchable text (title is most important, then description) - text = "" - if article.title: - text += article.title.lower() + " " - if article.description: - text += article.description.lower() + " " - - # Add first 500 chars of content if available - if article.normalized_content: - text += article.normalized_content[:500].lower() - - # Category weights (how strongly each keyword category signals relevance) - category_weights = { - "core": 3.0, # Core terms are strongest signal - "threats": 2.0, # Specific threats/technologies - "policy": 2.0, # Policy/governance terms - "geopolitical": 2.0, - "tech": 1.5, - "industry": 1.5, - "topics": 1.5, - "tools": 1.0, - "providers": 1.0, - "entities": 1.0, # Mentioned companies/orgs - } - - # Score based on keyword matches - for category, terms in keywords.items(): - weight = category_weights.get(category, 1.0) - matches = sum(1 for term in terms if term in text) - score += matches * weight - - # Entity matching (strong signal if relevant entities mentioned) - if article.entities and "entities" in keywords: - entity_keywords = keywords["entities"] - for entity in article.entities: - if any(kw in entity.lower() for kw in entity_keywords): - score += 3.0 # High confidence for entity match - - # Feed category matching - if article.feed and article.feed.category: - category_lower = article.feed.category.lower() - - # Check if topic or core keywords in feed category - if topic in category_lower: - score += 2.0 - elif "core" in keywords: - core_keywords = keywords["core"] - if any(kw in category_lower for kw in core_keywords): - score += 2.0 - - # Decision threshold: require minimum score - threshold = 3.0 - matches = score >= threshold - - if matches: - logger.debug( - f"Article '{article.title[:50]}...' matched topic '{topic}' with score {score:.1f}" - ) - - return matches, score - - def matches_scope(self, article: Article, scope: str, user_location: dict) -> bool: - """ - Check if article matches geographic scope - - Args: - article: Article to check - scope: Geographic scope ('local', 'state', 'national', 'global') - user_location: User's location dict (city, state, region) - - Returns: - bool - Whether article matches scope - """ - # Combine searchable text - text = "" - if article.title: - text += article.title.lower() + " " - if article.description: - text += article.description.lower() - - if scope == "local": - # Extract user location details - city = user_location.get("city", "").lower() - region = user_location.get("region", "").lower() - - # Check for direct mentions - if city and city in text: - return True - if region and region in text: - return True - - # Check entities for location matches - if article.entities: - for entity in article.entities: - entity_lower = entity.lower() - if (city and city in entity_lower) or (region and region in entity_lower): - return True - - # Check feed category for 'local' - return bool( - article.feed and article.feed.category and "local" in article.feed.category.lower() - ) - - elif scope == "state": - state = user_location.get("state", "").lower() - - # State name in text - if state and state in text: - return True - - # State in feed category - if article.feed and article.feed.category: - cat = article.feed.category.lower() - if state in cat or "state" in cat or "regional" in cat: - return True - - return False - - elif scope == "national": - national_keywords = [ - "federal", - "congress", - "senate", - "house of representatives", - "washington", - "white house", - "national", - "u.s.", - "usa", - "president", - "supreme court", - "fbi", - "cia", - "dhs", - "nationwide", - ] - - # Check for national keywords - if any(kw in text for kw in national_keywords): - return True - - # Check feed category - if article.feed and article.feed.category: - cat = article.feed.category.lower() - if any(kw in cat for kw in ["national", "federal", "us", "usa"]): - return True - - return False - - elif scope == "global": - # Global is more permissive - includes international news - global_keywords = [ - "international", - "global", - "worldwide", - "world", - "europe", - "asia", - "africa", - "united nations", - "nato", - "g7", - "g20", - "china", - "russia", - "india", - ] - - if any(kw in text for kw in global_keywords): - return True - - if article.feed and article.feed.category: - cat = article.feed.category.lower() - if any(kw in cat for kw in ["global", "international", "world"]): - return True - - return False - - # Unknown scope = no filtering - logger.warning(f"Unknown scope: {scope}") - return True - - def filter_articles( - self, articles: list[Article], topic_filters: dict, user_profile: UserProfile - ) -> list[Article]: - """ - Apply topic and scope filters to article list - - Args: - articles: List of Article objects to filter - topic_filters: Dict with 'topics' and/or 'scopes' keys - user_profile: UserProfile for location/context - - Returns: - Filtered list of articles sorted by relevance score - """ - filtered = articles.copy() - filter_stats = { - "input_count": len(articles), - "topics": topic_filters.get("topics", []), - "scopes": topic_filters.get("scopes", []), - } - - # Apply topic filters (OR logic within topics) - topics = topic_filters.get("topics", []) - if topics: - topic_matches = [] - for article in filtered: - for topic in topics: - matches, score = self.matches_topic(article, topic) - if matches: - topic_matches.append((article, score)) - break # Article matches at least one topic - - # Sort by score (descending) to prioritize best matches - topic_matches.sort(key=lambda x: x[1], reverse=True) - filtered = [article for article, score in topic_matches] - - filter_stats["after_topic_filter"] = len(filtered) - - # Apply scope filters (OR logic within scopes) - scopes = topic_filters.get("scopes", []) - if scopes: - user_location = user_profile.get_primary_location() - scope_matches = [] - - for article in filtered: - for scope in scopes: - if self.matches_scope(article, scope, user_location): - scope_matches.append(article) - break # Article matches at least one scope - - filtered = scope_matches - filter_stats["after_scope_filter"] = len(filtered) - - filter_stats["output_count"] = len(filtered) - - logger.info( - f"TopicMatcher: {filter_stats['output_count']}/{filter_stats['input_count']} " - f"articles matched filters {topic_filters}" - ) - - return filtered diff --git a/src/database/migrations/add_beat_entities.py b/src/database/migrations/add_beat_entities.py deleted file mode 100644 index a30d28c..0000000 --- a/src/database/migrations/add_beat_entities.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -Migration: Add Institutional Activity Tables -Adds BeatEntity and EntityMention so a beat can record which of its declared -institutions appeared in each run. - -Purely additive (2026-08-26): no existing table is altered, so an unmigrated -database keeps working for every other command and this migration cannot lose -data. Requires the beat tables from add_beats.py, which both new tables key -off. - -There is no persons table here and no place for one: `beat_entities.kind` -carries `org`, `program` or `document_type` only. See -backlog/006-institutional-activity.md. -""" - -from src.database.connection import engine -from src.database.models import BeatEntity, EntityMention - - -def upgrade(): - """Create institutional activity tables.""" - print("Creating institutional activity tables...") - - BeatEntity.__table__.create(engine, checkfirst=True) - print(" beat_entities table created") - - EntityMention.__table__.create(engine, checkfirst=True) - print(" entity_mentions table created") - - print("\nInstitutional activity migration completed.") - - -def downgrade(): - """Drop institutional activity tables.""" - print("Dropping institutional activity tables...") - - EntityMention.__table__.drop(engine, checkfirst=True) - print(" entity_mentions table dropped") - - BeatEntity.__table__.drop(engine, checkfirst=True) - print(" beat_entities table dropped") - - print("\nInstitutional activity downgrade completed.") - - -if __name__ == "__main__": - import sys - - if len(sys.argv) > 1 and sys.argv[1] == "down": - downgrade() - else: - upgrade() diff --git a/src/database/migrations/add_beats.py b/src/database/migrations/add_beats.py deleted file mode 100644 index ef88307..0000000 --- a/src/database/migrations/add_beats.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Migration: Add Beat Tables -Adds Beat and BeatRun tables so brief runs can be attributed to a subject. - -Purely additive (2026-08-26): no existing table is altered, so an unmigrated -database keeps working for every non-beat command and this migration cannot -lose data. -""" - -from src.database.connection import engine -from src.database.models import Beat, BeatRun - - -def upgrade(): - """Create beat tables.""" - print("Creating beat tables...") - - Beat.__table__.create(engine, checkfirst=True) - print(" beats table created") - - BeatRun.__table__.create(engine, checkfirst=True) - print(" beat_runs table created") - - print("\nBeat migration completed.") - - -def downgrade(): - """Drop beat tables.""" - print("Dropping beat tables...") - - BeatRun.__table__.drop(engine, checkfirst=True) - print(" beat_runs table dropped") - - Beat.__table__.drop(engine, checkfirst=True) - print(" beats table dropped") - - print("\nBeat downgrade completed.") - - -if __name__ == "__main__": - import sys - - if len(sys.argv) > 1 and sys.argv[1] == "down": - downgrade() - else: - upgrade() diff --git a/src/database/migrations/add_calibration_loop.py b/src/database/migrations/add_calibration_loop.py deleted file mode 100644 index 0cc3a8e..0000000 --- a/src/database/migrations/add_calibration_loop.py +++ /dev/null @@ -1,261 +0,0 @@ -""" -Migration: Add the operator calibration loop (2026-08-27, backlog task 011) - -Two tables change, both preserving every existing row: - -``questions`` gains ``cadence`` and ``last_reviewed_at``. Purely additive -- -two nullable columns via ALTER TABLE. Every pre-existing question keeps a null -cadence, which is correct: it was surfaced by the model from coverage, not put -on a review schedule by the operator, so ``forecast --due`` never raises it. - -``predictions`` gains ``author``, ``due_by``, ``confidence`` and ``outcome``, -and relaxes ``made_in_synthesis_id`` to nullable so a claim the operator stakes -from the CLI -- which has no synthesis behind it -- can be stored at all. -SQLite cannot drop a NOT NULL constraint in place, so this half is the standard -SQLite table rebuild: rename, recreate from the model, copy every row, drop the -old table. Every existing prediction backfills to ``author='model'``, which is -what all of them were. - -Reversible. ``downgrade`` restores both tables to their pre-011 shape, and -refuses rather than destroying data if any operator prediction exists: an -operator-staked claim has no representation in the old schema, so silently -dropping or relabelling one would be a lie about who said it. -""" - -from sqlalchemy import inspect, text - -from src.database.connection import engine -from src.database.models import Prediction, Question - -# The pre-011 shape of `predictions`, needed verbatim by downgrade(). Kept as a -# literal rather than derived from the model, because the model has moved on. -_PREDICTIONS_PRE_011_DDL = """ -CREATE TABLE predictions ( - id INTEGER NOT NULL, - question_id INTEGER NOT NULL, - observable_text TEXT NOT NULL, - trigger_condition TEXT NOT NULL, - made_at DATETIME NOT NULL, - made_in_synthesis_id INTEGER NOT NULL, - status VARCHAR(20) NOT NULL, - resolved_at DATETIME, - resolution_note TEXT, - PRIMARY KEY (id), - FOREIGN KEY(question_id) REFERENCES questions (id), - FOREIGN KEY(made_in_synthesis_id) REFERENCES narrative_syntheses (id) -) -""" - -_PREDICTIONS_PRE_011_COLUMNS = [ - "id", - "question_id", - "observable_text", - "trigger_condition", - "made_at", - "made_in_synthesis_id", - "status", - "resolved_at", - "resolution_note", -] - -_PREDICTIONS_PRE_011_INDEXES = [ - "CREATE INDEX idx_prediction_status ON predictions (status)", - "CREATE INDEX idx_prediction_question ON predictions (question_id)", - "CREATE INDEX idx_prediction_made_at ON predictions (made_at)", -] - -# Every index on `predictions`, old and new. Dropped before a rebuild so the -# names are free: SQLite keeps indexes attached to a table across a RENAME, -# under their original names, which would collide on recreate. -_ALL_PREDICTION_INDEXES = [ - "idx_prediction_status", - "idx_prediction_question", - "idx_prediction_made_at", - "idx_prediction_author", - "idx_prediction_due_by", -] - -_QUESTION_COLUMNS = { - "cadence": "VARCHAR(20)", - "last_reviewed_at": "DATETIME", -} - - -def _column_names(conn, table: str) -> set[str]: - return {col["name"] for col in inspect(conn).get_columns(table)} - - -def _has_table(conn, table: str) -> bool: - return inspect(conn).has_table(table) - - -def _require_sqlite(conn) -> None: - """ - The rebuild half is SQLite-specific. - - Failing loudly beats emitting DDL that silently does the wrong thing on - another engine. InsightWeaver has only ever run on SQLite. - """ - if conn.dialect.name != "sqlite": - raise RuntimeError( - f"This migration rebuilds the predictions table using SQLite semantics; " - f"the configured database is '{conn.dialect.name}'. Port it before running." - ) - - -def upgrade(): - """Add the cadence and authorship columns, preserving every row.""" - with engine.begin() as conn: - _require_sqlite(conn) - - print("Adding review cadence to questions...") - if not _has_table(conn, "questions"): - Question.__table__.create(conn, checkfirst=True) - print(" questions table created") - else: - existing = _column_names(conn, "questions") - for name, sql_type in _QUESTION_COLUMNS.items(): - if name in existing: - print(f" questions.{name} already present, skipping") - continue - conn.execute(text(f"ALTER TABLE questions ADD COLUMN {name} {sql_type}")) - print(f" questions.{name} added") - - print("\nAdding authorship, due date and outcome to predictions...") - if not _has_table(conn, "predictions"): - Prediction.__table__.create(conn, checkfirst=True) - print(" predictions table created") - elif "author" in _column_names(conn, "predictions"): - print(" predictions already carries author, skipping rebuild") - else: - _rebuild_predictions_forward(conn) - - print("\nCalibration loop migration completed.") - - -def _rebuild_predictions_forward(conn) -> None: - """ - SQLite table rebuild: relax made_in_synthesis_id and add the new columns. - - Row count is asserted on both sides. A rebuild that quietly lost rows would - corrupt the one ledger this task exists to make trustworthy. - """ - before = conn.execute(text("SELECT COUNT(*) FROM predictions")).scalar_one() - print(f" rebuilding predictions ({before} row(s) to carry over)") - - for index_name in _ALL_PREDICTION_INDEXES: - conn.execute(text(f"DROP INDEX IF EXISTS {index_name}")) - conn.execute(text("ALTER TABLE predictions RENAME TO predictions_pre_011")) - - Prediction.__table__.create(conn, checkfirst=False) - - carried = ", ".join(_PREDICTIONS_PRE_011_COLUMNS) - conn.execute( - text( - f"INSERT INTO predictions ({carried}, author) " - f"SELECT {carried}, 'model' FROM predictions_pre_011" - ) - ) - - after = conn.execute(text("SELECT COUNT(*) FROM predictions")).scalar_one() - if after != before: - raise RuntimeError( - f"Rebuild carried {after} of {before} predictions. Rolling back; " - "the database is unchanged." - ) - - conn.execute(text("DROP TABLE predictions_pre_011")) - print(f" {after} prediction(s) backfilled to author='model'") - - -def downgrade(): - """Restore the pre-011 shape of both tables.""" - with engine.begin() as conn: - _require_sqlite(conn) - - print("Reverting predictions...") - if not _has_table(conn, "predictions"): - print(" no predictions table, nothing to revert") - elif "author" not in _column_names(conn, "predictions"): - print(" predictions already at the pre-011 shape, skipping") - else: - _rebuild_predictions_back(conn) - - print("\nReverting questions...") - if not _has_table(conn, "questions"): - print(" no questions table, nothing to revert") - else: - existing = _column_names(conn, "questions") - for name in _QUESTION_COLUMNS: - if name not in existing: - print(f" questions.{name} already absent, skipping") - continue - conn.execute(text(f"ALTER TABLE questions DROP COLUMN {name}")) - print(f" questions.{name} dropped") - - print("\nCalibration loop downgrade completed.") - - -def _rebuild_predictions_back(conn) -> None: - """ - Rebuild predictions to the pre-011 shape, or refuse. - - An operator prediction has no synthesis and no author column to live in - once this runs, so it cannot survive the revert. Refusing is the honest - answer; quietly relabelling it 'model' would put words in the operator's - mouth, and dropping it would delete the record silently. - """ - stakes = conn.execute( - text("SELECT COUNT(*) FROM predictions WHERE author = 'operator'") - ).scalar_one() - if stakes: - raise RuntimeError( - f"{stakes} operator prediction(s) are in the ledger. The pre-011 schema " - "has nowhere to record who staked a claim or when it resolves, so this " - "downgrade would silently misattribute them. Export or delete them " - "first if you really mean to revert. Nothing was changed." - ) - - orphans = conn.execute( - text("SELECT COUNT(*) FROM predictions WHERE made_in_synthesis_id IS NULL") - ).scalar_one() - if orphans: - raise RuntimeError( - f"{orphans} prediction(s) have no synthesis, which the pre-011 schema " - "requires. Nothing was changed." - ) - - before = conn.execute(text("SELECT COUNT(*) FROM predictions")).scalar_one() - print(f" rebuilding predictions ({before} row(s) to carry over)") - - for index_name in _ALL_PREDICTION_INDEXES: - conn.execute(text(f"DROP INDEX IF EXISTS {index_name}")) - conn.execute(text("ALTER TABLE predictions RENAME TO predictions_post_011")) - - conn.execute(text(_PREDICTIONS_PRE_011_DDL)) - for statement in _PREDICTIONS_PRE_011_INDEXES: - conn.execute(text(statement)) - - carried = ", ".join(_PREDICTIONS_PRE_011_COLUMNS) - conn.execute( - text(f"INSERT INTO predictions ({carried}) SELECT {carried} FROM predictions_post_011") - ) - - after = conn.execute(text("SELECT COUNT(*) FROM predictions")).scalar_one() - if after != before: - raise RuntimeError( - f"Revert carried {after} of {before} predictions. Rolling back; " - "the database is unchanged." - ) - - conn.execute(text("DROP TABLE predictions_post_011")) - print(f" {after} prediction(s) restored") - - -if __name__ == "__main__": - import sys - - if len(sys.argv) > 1 and sys.argv[1] == "down": - downgrade() - else: - upgrade() diff --git a/src/database/migrations/add_decisions.py b/src/database/migrations/add_decisions.py deleted file mode 100644 index 747b8ee..0000000 --- a/src/database/migrations/add_decisions.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -Migration: Add Decision Journal Tables -Adds Decision, DecisionFactor, and DecisionEvidence. After creating the -tables, seeds them from personal_priorities.active_decisions in the user -profile if the decisions table is empty -- the profile is the seed, the -database is the source of truth from then on. -""" - -import logging - -from src.database.connection import engine, get_db -from src.database.models import Decision, DecisionEvidence, DecisionFactor - -logger = logging.getLogger(__name__) - - -def _seed_from_profile(): - """Seed decisions from the user profile if the table is empty.""" - try: - from src.utils.profile_loader import get_user_profile - - profile = get_user_profile() - active = profile.get_active_decisions() - except FileNotFoundError: - print(" no user profile found; skipping decision seeding") - return - except Exception as e: - print(f" could not read profile decisions; skipping seeding ({e})") - return - - if not active: - print(" profile has no active_decisions; nothing to seed") - return - - with get_db() as session: - if session.query(Decision).count() > 0: - print(" decisions table already populated; skipping seeding") - return - - for entry in active: - decision = Decision( - name=entry.get("name", "Unnamed decision"), - decision_type=entry.get("decision_type", "other"), - ) - session.add(decision) - session.flush() - for factor_name in entry.get("key_factors", []): - session.add(DecisionFactor(decision_id=decision.id, name=factor_name)) - session.commit() - print(f" seeded {len(active)} decisions from profile") - - -def upgrade(): - """Create decision journal tables and seed from the profile.""" - print("Creating decision journal tables...") - Decision.__table__.create(engine, checkfirst=True) - print(" decisions table created") - DecisionFactor.__table__.create(engine, checkfirst=True) - print(" decision_factors table created") - DecisionEvidence.__table__.create(engine, checkfirst=True) - print(" decision_evidence table created") - - _seed_from_profile() - print("\nDecision journal migration completed.") - - -def downgrade(): - """Drop decision journal tables.""" - print("Dropping decision journal tables...") - DecisionEvidence.__table__.drop(engine, checkfirst=True) - print(" decision_evidence table dropped") - DecisionFactor.__table__.drop(engine, checkfirst=True) - print(" decision_factors table dropped") - Decision.__table__.drop(engine, checkfirst=True) - print(" decisions table dropped") - print("\nDecision journal downgrade completed.") - - -if __name__ == "__main__": - import sys - - if len(sys.argv) > 1 and sys.argv[1] == "down": - downgrade() - else: - upgrade() diff --git a/src/database/migrations/add_frame_tables.py b/src/database/migrations/add_frame_tables.py deleted file mode 100644 index c7af737..0000000 --- a/src/database/migrations/add_frame_tables.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -Migration: Add Narrative Frame Tables -Adds TopicCluster, NarrativeFrame, ArticleFrame, and FrameGap tables -for emergent frame discovery and tracking. -""" - -from src.database.connection import engine -from src.database.models import ArticleFrame, FrameGap, NarrativeFrame, TopicCluster - - -def upgrade(): - """Create narrative frame tables.""" - print("Creating narrative frame tables...") - - TopicCluster.__table__.create(engine, checkfirst=True) - print(" topic_clusters table created") - - NarrativeFrame.__table__.create(engine, checkfirst=True) - print(" narrative_frames table created") - - ArticleFrame.__table__.create(engine, checkfirst=True) - print(" article_frames table created") - - FrameGap.__table__.create(engine, checkfirst=True) - print(" frame_gaps table created") - - print("\nNarrative frame tables migration completed.") - - -def downgrade(): - """Drop narrative frame tables.""" - print("Dropping narrative frame tables...") - - FrameGap.__table__.drop(engine, checkfirst=True) - print(" frame_gaps table dropped") - - ArticleFrame.__table__.drop(engine, checkfirst=True) - print(" article_frames table dropped") - - NarrativeFrame.__table__.drop(engine, checkfirst=True) - print(" narrative_frames table dropped") - - TopicCluster.__table__.drop(engine, checkfirst=True) - print(" topic_clusters table dropped") - - print("\nNarrative frame tables downgrade completed.") - - -if __name__ == "__main__": - upgrade() diff --git a/src/database/migrations/add_predictions.py b/src/database/migrations/add_predictions.py deleted file mode 100644 index a3cdc10..0000000 --- a/src/database/migrations/add_predictions.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -Migration: Add Predictions Ledger -Adds the Prediction table for tracking falsifiable observables across runs. -""" - -from src.database.connection import engine -from src.database.models import Prediction - - -def upgrade(): - """Create the predictions table.""" - print("Creating predictions table...") - Prediction.__table__.create(engine, checkfirst=True) - print(" predictions table created") - print("\nPredictions ledger migration completed.") - - -def downgrade(): - """Drop the predictions table.""" - print("Dropping predictions table...") - Prediction.__table__.drop(engine, checkfirst=True) - print(" predictions table dropped") - print("\nPredictions ledger downgrade completed.") - - -if __name__ == "__main__": - import sys - - if len(sys.argv) > 1 and sys.argv[1] == "down": - downgrade() - else: - upgrade() diff --git a/src/database/migrations/add_questions.py b/src/database/migrations/add_questions.py deleted file mode 100644 index 46705e5..0000000 --- a/src/database/migrations/add_questions.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Migration: Add Question Graph Tables -Adds Question and QuestionSituation tables for cross-run question tracking. -""" - -from src.database.connection import engine -from src.database.models import Question, QuestionSituation - - -def upgrade(): - """Create question graph tables.""" - print("Creating question graph tables...") - - Question.__table__.create(engine, checkfirst=True) - print(" questions table created") - - QuestionSituation.__table__.create(engine, checkfirst=True) - print(" question_situations table created") - - print("\nQuestion graph migration completed.") - - -def downgrade(): - """Drop question graph tables.""" - print("Dropping question graph tables...") - - QuestionSituation.__table__.drop(engine, checkfirst=True) - print(" question_situations table dropped") - - Question.__table__.drop(engine, checkfirst=True) - print(" questions table dropped") - - print("\nQuestion graph downgrade completed.") - - -if __name__ == "__main__": - import sys - - if len(sys.argv) > 1 and sys.argv[1] == "down": - downgrade() - else: - upgrade() diff --git a/src/database/migrations/add_standing_questions.py b/src/database/migrations/add_standing_questions.py deleted file mode 100644 index 48e74ae..0000000 --- a/src/database/migrations/add_standing_questions.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Migration: Add the beat standing-question table -Adds BeatStandingQuestion so a beat can declare what it is watching. - -Purely additive (2026-08-26, backlog task 007): no existing table is altered. -A database that has run add_beats but not this migration keeps working for -every command except `brief --beat` on a beat that declares standing -questions, which fails loudly naming this migration rather than dropping the -declaration. -""" - -from src.database.connection import engine -from src.database.models import BeatStandingQuestion - - -def upgrade(): - """Create the standing-question table.""" - print("Creating standing question table...") - - BeatStandingQuestion.__table__.create(engine, checkfirst=True) - print(" beat_standing_questions table created") - - print("\nStanding question migration completed.") - - -def downgrade(): - """Drop the standing-question table.""" - print("Dropping standing question table...") - - BeatStandingQuestion.__table__.drop(engine, checkfirst=True) - print(" beat_standing_questions table dropped") - - print("\nStanding question downgrade completed.") - - -if __name__ == "__main__": - import sys - - if len(sys.argv) > 1 and sys.argv[1] == "down": - downgrade() - else: - upgrade() diff --git a/src/database/migrations/drop_briefing_tables.py b/src/database/migrations/drop_briefing_tables.py new file mode 100644 index 0000000..facfab1 --- /dev/null +++ b/src/database/migrations/drop_briefing_tables.py @@ -0,0 +1,307 @@ +""" +Migration: drop the tables belonging to the deleted briefing product. + +Backlog task 012 deleted the briefing product -- synthesis, rendering, topic +clusters, narrative frames, questions, predictions, decisions and beats. Their +models are gone from ``src/database/models.py``; this drops the tables so that +the schema and the code agree, rather than leaving nineteen unreferenced tables +that read as current to the next person who opens the database. + +**This destroys real data.** Measured on the operator's database 2026-08-31: +46 questions, 42 question_situations, 33 predictions, 42 narrative_frames, +39 frame_gaps, 337 provenance_records, 20 article_frames, 14 topic_clusters, +17 beat_entities, 17 entity_mentions, plus the syntheses and runs. That is the +whole record of the product being removed, and none of it is reconstructible +from the articles that remain. + +Two guards, because a migration that is one keystroke from unrecoverable should +not be one keystroke from *accidentally* unrecoverable: + +1. It refuses to run without ``--confirm``. There is no prompt to hold Enter + through and no environment variable that quietly enables it in CI. +2. It writes every target table -- schema and rows -- to a timestamped SQL dump + before dropping anything, and refuses to drop if the dump cannot be written. + +**What "reversible" means here.** ``downgrade()`` recreates the dropped tables +from their captured DDL, so the *schema* round-trips. It does not restore rows, +because a dropped row is gone; the dump from step 2 is the only path back to the +data, and restoring it is a deliberate act (``--restore``), not an automatic +one. Saying a destructive migration is "reversible" without that distinction is +the kind of claim this file exists not to make. + +``articles`` and ``rss_feeds`` are never touched. ``articles`` holds 55,249 rows +and its fate is backlog task 014's decision, not this migration's -- it is +listed in :data:`PROTECTED_TABLES` and the drop asserts against it. + +Added 2026-08-31 for backlog task 012. +""" + +from __future__ import annotations + +import argparse +import sys +from datetime import datetime +from pathlib import Path + +from sqlalchemy import text +from sqlalchemy.engine import Engine + +from src.database.connection import engine + +__all__ = [ + "BRIEFING_TABLES", + "PROTECTED_TABLES", + "capture", + "downgrade", + "dump_path", + "upgrade", +] + +# Every table that belonged to a concept task 012 deleted, in dependency order +# (children before parents) so the drops do not trip a foreign key. +BRIEFING_TABLES: tuple[str, ...] = ( + # Frames hang off topic clusters and articles. + "article_frames", + "frame_gaps", + "narrative_frames", + "topic_clusters", + # Beat entity observations hang off beats and beat runs. + "entity_mentions", + "beat_entities", + "beat_standing_questions", + "beat_runs", + "beats", + # Decisions. + "decision_evidence", + "decision_factors", + "decisions", + # The question graph and everything keyed off it. + "predictions", + "question_situations", + "questions", + # Synthesis outputs and their provenance. + "provenance_records", + "narrative_syntheses", + "context_snapshots", + "analysis_runs", +) + +# Tables this migration must never touch, asserted at runtime rather than left +# to the correctness of the list above. `articles` is task 014's decision. +PROTECTED_TABLES: frozenset[str] = frozenset({"articles", "rss_feeds"}) + +_CONFIRM_HELP = ( + "This migration permanently drops the briefing tables and the rows in them.\n" + "Re-run with --confirm once you have read what it will drop:\n\n" + " python -m src.database.migrations.drop_briefing_tables --confirm\n" +) + + +def _guard() -> None: + """Fail loudly if the drop list and the protected list ever overlap.""" + collision = PROTECTED_TABLES & set(BRIEFING_TABLES) + if collision: + raise AssertionError( + f"refusing to run: protected table(s) {sorted(collision)} appear in the drop list" + ) + + +def _existing(conn, tables: tuple[str, ...]) -> list[str]: + """Which of ``tables`` this database actually has, in drop order.""" + present = { + row[0] for row in conn.execute(text("SELECT name FROM sqlite_master WHERE type='table'")) + } + return [t for t in tables if t in present] + + +def dump_path(db_url: str, when: datetime | None = None) -> Path: + """ + Where the pre-drop dump goes: beside the database, stamped with the time. + + Beside the database rather than in the repo, because the dump contains the + operator's data and the repo is a place things get committed from. + """ + stamp = (when or datetime.now()).strftime("%Y%m%d-%H%M%S") + if db_url.startswith("sqlite:///"): + db_file = Path(db_url.removeprefix("sqlite:///")) + return db_file.parent / f"{db_file.name}.briefing-tables-{stamp}.sql" + return Path.cwd() / f"briefing-tables-{stamp}.sql" + + +def capture(target: Engine, out: Path) -> tuple[Path, dict[str, int]]: + """ + Write the schema and every row of the briefing tables to ``out``. + + Returns the path written and a ``{table: row_count}`` map. Raises if the + file cannot be written -- a drop whose backup silently failed is the exact + accident this function exists to prevent. + """ + counts: dict[str, int] = {} + lines: list[str] = [ + "-- Pre-drop capture of the briefing tables (backlog task 012).", + f"-- Written {datetime.now().isoformat(timespec='seconds')} from {target.url}.", + "-- Restore with: python -m src.database.migrations.drop_briefing_tables" + " --restore ", + "", + ] + with target.connect() as conn: + for table in _existing(conn, BRIEFING_TABLES): + ddl = conn.execute( + text("SELECT sql FROM sqlite_master WHERE type='table' AND name=:n"), + {"n": table}, + ).scalar() + lines.append(f"-- {table}") + if ddl: + lines.append(f"{ddl};") + for idx_sql in conn.execute( + text( + "SELECT sql FROM sqlite_master WHERE type='index' AND tbl_name=:n" + " AND sql IS NOT NULL" + ), + {"n": table}, + ): + lines.append(f"{idx_sql[0]};") + + rows = conn.execute(text(f'SELECT * FROM "{table}"')) # noqa: S608 - fixed list + cols = list(rows.keys()) + count = 0 + for row in rows: + values = ", ".join(_sql_literal(v) for v in row) + collist = ", ".join(f'"{c}"' for c in cols) + lines.append(f'INSERT INTO "{table}" ({collist}) VALUES ({values});') + count += 1 + counts[table] = count + lines.append("") + + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines), encoding="utf-8") + return out, counts + + +def _sql_literal(value: object) -> str: + """One Python value as a SQLite literal.""" + if value is None: + return "NULL" + if isinstance(value, bool): + return "1" if value else "0" + if isinstance(value, int | float): + return repr(value) + escaped = str(value).replace("'", "''") + return f"'{escaped}'" + + +def _statements(body: str) -> list[str]: + """ + Split a capture file into executable statements. + + Comment lines are stripped *before* splitting, not after. Doing it the other + way round silently drops the first statement of every table: each ``CREATE + TABLE`` is preceded by a ``-- `` marker, so the chunk it lands in + starts with a comment and a naive "does this chunk start with CREATE" filter + discards it. That produced a downgrade which recreated nothing and a restore + which then failed on the first INSERT. + """ + stripped = "\n".join(line for line in body.splitlines() if not line.lstrip().startswith("--")) + return [s.strip() for s in stripped.split(";\n") if s.strip()] + + +def upgrade(target: Engine | None = None, *, confirmed: bool = False) -> dict[str, int]: + """ + Drop every briefing table, after capturing it. + + ``confirmed`` must be True. The flag is a parameter rather than a prompt so + that the refusal is testable and so that no interactive path exists to + fat-finger through. + """ + _guard() + if not confirmed: + raise SystemExit(_CONFIRM_HELP) + + target = target or engine + out, counts = capture(target, dump_path(str(target.url))) + total = sum(counts.values()) + print(f"Captured {total} row(s) across {len(counts)} table(s) to {out}") + + with target.begin() as conn: + present = _existing(conn, BRIEFING_TABLES) + for table in present: + if table in PROTECTED_TABLES: # pragma: no cover - _guard covers this + raise AssertionError(f"refusing to drop protected table {table}") + conn.execute(text(f'DROP TABLE IF EXISTS "{table}"')) + print(f" dropped {table} ({counts.get(table, 0)} row(s))") + + print(f"\nDropped {len(present)} briefing table(s). Data is recoverable only from {out}.") + return counts + + +def downgrade(target: Engine | None = None, dump: Path | None = None) -> list[str]: + """ + Recreate the dropped tables from a capture file, schema only. + + Rows are **not** restored -- use ``--restore`` for that. This exists so the + schema change round-trips; it cannot undo the data loss and does not claim + to. + """ + target = target or engine + if dump is None: + raise SystemExit( + "downgrade needs the capture file written by upgrade:\n" + " python -m src.database.migrations.drop_briefing_tables --down \n" + ) + statements = [ + s + for s in _statements(dump.read_text(encoding="utf-8")) + if s.upper().startswith(("CREATE TABLE", "CREATE INDEX", "CREATE UNIQUE")) + ] + recreated: list[str] = [] + with target.begin() as conn: + for stmt in statements: + conn.execute(text(stmt)) + recreated.append(stmt.split()[2].strip('"')) + print(f"Recreated {len(recreated)} object(s), empty. Rows were not restored.") + return recreated + + +def restore(target: Engine | None = None, dump: Path | None = None) -> int: + """Replay a capture file in full, schema and rows. Deliberate and explicit.""" + target = target or engine + if dump is None: + raise SystemExit("--restore needs a capture file") + applied = 0 + with target.begin() as conn: + for stmt in _statements(dump.read_text(encoding="utf-8")): + conn.execute(text(stmt)) + applied += 1 + print(f"Replayed {applied} statement(s) from {dump}.") + return applied + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="drop_briefing_tables", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--confirm", + action="store_true", + help="actually drop the tables (required; there is no prompt)", + ) + parser.add_argument("--down", type=Path, metavar="DUMP", help="recreate schema from DUMP") + parser.add_argument( + "--restore", type=Path, metavar="DUMP", help="replay DUMP in full, schema and rows" + ) + args = parser.parse_args(argv) + + if args.restore: + restore(dump=args.restore) + return 0 + if args.down: + downgrade(dump=args.down) + return 0 + upgrade(confirmed=args.confirm) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/database/models.py b/src/database/models.py index 4f2ec7d..4eac0be 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -1,6 +1,18 @@ """ -Optimized Database Models for Context Engineering -Simplified schema focusing on context curation and synthesis outputs +Database models. + +Two tables: the feeds we read and the articles we stored from them. + +Everything else was deleted by backlog task 012 along with the briefing product +that owned it -- syntheses, context snapshots, provenance, topic clusters, +narrative frames, questions, predictions, decisions and beats. The tables are +dropped by ``src.database.migrations.drop_briefing_tables``; the models are +removed here so that ``create_tables()`` cannot quietly recreate a concept the +rewrite removed. + +``articles`` is deliberately untouched. It holds the corpus (55,249 rows as of +2026-08-31) and what becomes of it is backlog task 014's decision, not this +one's. """ from sqlalchemy import ( @@ -95,689 +107,10 @@ class Article(Base): Index("idx_relevance_score", "relevance_score"), # For context selection Index("idx_filtered", "filtered"), # Quick filtering queries # Composite indexes for critical query paths - Index( - "idx_articles_filtered_fetched", "filtered", "fetched_at" - ), # curator.py synthesis queries - Index( - "idx_articles_filtered_published", "filtered", "published_date" - ), # content_engine.py queries - ) - - -class AnalysisRun(Base): - """ - Execution tracking for synthesis runs - Simplified for context engineering approach - """ - - __tablename__ = "analysis_runs" - - id = Column(Integer, primary_key=True) - run_type = Column(String(50)) # 'narrative_synthesis' - status = Column(String(50)) # 'started', 'completed', 'failed' - started_at = Column(DateTime, default=utcnow) - completed_at = Column(DateTime) - articles_processed = Column(Integer) - context_token_count = Column(Integer) # Approximate tokens sent to Claude - claude_model = Column(String(100)) # Track which model was used - error_message = Column(Text) - - __table_args__ = (Index("idx_run_started_at", "started_at"),) - - -class NarrativeSynthesis(Base): - """ - Primary output: Claude-generated narrative intelligence briefs - Contains all analysis (trends, predictions, insights) in structured JSON - """ - - __tablename__ = "narrative_syntheses" - - id = Column(Integer, primary_key=True) - analysis_run_id = Column(Integer, ForeignKey("analysis_runs.id")) - context_snapshot_id = Column(Integer, ForeignKey("context_snapshots.id")) - - # User context - user_profile_version = Column(String(50)) - - # Synthesis output - synthesis_data = Column(JSON) # Full structured output from Claude - executive_summary = Column(Text) # Extracted for quick access - - # Metadata - articles_analyzed = Column(Integer) - temporal_scope = Column(String(100)) # "immediate,near,medium,long" - generated_at = Column(DateTime, default=utcnow) - - __table_args__ = ( - Index("idx_narrative_generated_at", "generated_at"), - Index("idx_narrative_profile_version", "user_profile_version"), - ) - - -class ContextSnapshot(Base): - """ - Stores exact context sent to Claude for each synthesis - Enables reproducibility and debugging - """ - - __tablename__ = "context_snapshots" - - id = Column(Integer, primary_key=True) - synthesis_id = Column(Integer) # Will reference narrative_syntheses after creation - - # Context composition - article_ids = Column(JSON) # List of article IDs included - context_size_tokens = Column(Integer) # Approximate token count - user_profile_hash = Column(String(64)) # Hash of user profile used - - # Context metadata - historical_summaries = Column(Text) # Memory context included - instructions = Column(Text) # Instructions sent to Claude - - created_at = Column(DateTime, default=utcnow) - - __table_args__ = (Index("idx_context_created_at", "created_at"),) - - -class ProvenanceRecord(Base): - """ - Provenance chain for a synthesis claim - Shows the reasoning path from sources to conclusions - """ - - __tablename__ = "provenance_records" - - id = Column(Integer, primary_key=True) - synthesis_id = Column(Integer, ForeignKey("narrative_syntheses.id"), nullable=False) - - # Claim being traced - claim_text = Column(Text, nullable=False) - claim_location = Column(String(100)) - - # Source attribution - contributing_sources = Column(JSON, nullable=False) - confidence_breakdown = Column(JSON) - alternative_interpretations = Column(JSON, default=list) - reasoning_chain = Column(JSON) - - created_at = Column(DateTime, default=utcnow) - - __table_args__ = ( - Index("idx_provenance_synthesis", "synthesis_id"), - Index("idx_provenance_created", "created_at"), - ) - - -# ============================================================================ -# Narrative Frame Glossary -# Emergent frame discovery from corpus behavior -# ============================================================================ - - -class TopicCluster(Base): - """ - A cluster of articles about a related topic. - Frames are discovered and tracked per cluster. - """ - - __tablename__ = "topic_clusters" - - id = Column(Integer, primary_key=True) - name = Column(String(200), nullable=False) - keywords = Column(JSON, nullable=False) # List of keywords identifying this cluster - created_at = Column(DateTime, default=utcnow) - - frames = relationship("NarrativeFrame", back_populates="topic_cluster") - gaps = relationship("FrameGap", back_populates="topic_cluster") - - __table_args__ = (Index("idx_topic_cluster_name", "name"),) - - -class NarrativeFrame(Base): - """ - A distinct way of understanding a topic: what it emphasizes, - what it de-emphasizes, and what assumption it takes for granted. - Discovered emergently from the article corpus, validated by the user. - """ - - __tablename__ = "narrative_frames" - - id = Column(Integer, primary_key=True) - topic_cluster_id = Column(Integer, ForeignKey("topic_clusters.id"), nullable=False) - label = Column(String(200), nullable=False) - description = Column(Text) - assumptions = Column(Text) # What this frame takes for granted - first_seen = Column(DateTime, default=utcnow) - validated = Column(Boolean, default=False) # User has reviewed and accepted - - topic_cluster = relationship("TopicCluster", back_populates="frames") - article_frames = relationship("ArticleFrame", back_populates="frame") - - __table_args__ = ( - Index("idx_narrative_frame_cluster", "topic_cluster_id"), - Index("idx_narrative_frame_validated", "validated"), - ) - - -class ArticleFrame(Base): - """ - Maps an article to a narrative frame with a confidence score. - Populated during synthesis when known frames exist for a topic. - """ - - __tablename__ = "article_frames" - - id = Column(Integer, primary_key=True) - article_id = Column(Integer, ForeignKey("articles.id"), nullable=False) - frame_id = Column(Integer, ForeignKey("narrative_frames.id"), nullable=False) - confidence = Column(Float, nullable=False) # 0.0-1.0 - run_date = Column(DateTime, default=utcnow) - - frame = relationship("NarrativeFrame", back_populates="article_frames") - - __table_args__ = ( - Index("idx_article_frame_article", "article_id"), - Index("idx_article_frame_frame", "frame_id"), - Index("idx_article_frame_run_date", "run_date"), - ) - - -class FrameGap(Base): - """ - A recurring absence of a known frame from daily coverage. - Gaps are a feed curation signal: if a frame is consistently absent, - the user may want to add a source that carries that perspective. - """ - - __tablename__ = "frame_gaps" - - id = Column(Integer, primary_key=True) - topic_cluster_id = Column(Integer, ForeignKey("topic_clusters.id"), nullable=False) - frame_label = Column(String(200), nullable=False) # May reference a NarrativeFrame or be novel - first_detected = Column(DateTime, default=utcnow) - occurrences = Column(Integer, default=1) - feed_suggestion = Column(Text) # Suggested feed type to fill this gap - - topic_cluster = relationship("TopicCluster", back_populates="gaps") - - __table_args__ = ( - Index("idx_frame_gap_cluster", "topic_cluster_id"), - Index("idx_frame_gap_occurrences", "occurrences"), - ) - - -# ============================================================================ -# Question Graph -# Persistent, addressable questions that accumulate across runs. -# A Question is the unit that joins situations, predictions, and decisions -# across time. unresolved_question fields from synthesis output bind here. -# ============================================================================ - - -QUESTION_STATUS_OPEN = "open" -QUESTION_STATUS_RESOLVED = "resolved" -QUESTION_STATUS_SUPERSEDED = "superseded" - - -class Question(Base): - """ - A question the coverage is implicitly tracking across runs. - - Identity rule: when a synthesis emits an unresolved_question, the matcher - either binds it to an existing open Question or creates a new one. Resolved - questions are never auto-reopened; a fresh appearance gets a new Question - with previous_question_id pointing at the resolved record. - """ - - __tablename__ = "questions" - - id = Column(Integer, primary_key=True) - text = Column(Text, nullable=False) - normalized_text = Column(Text, nullable=False) - - first_asked_at = Column(DateTime, default=utcnow, nullable=False) - status = Column(String(20), nullable=False, default=QUESTION_STATUS_OPEN) - resolved_at = Column(DateTime, nullable=True) - resolution_note = Column(Text, nullable=True) - - # When a previously resolved question reappears, the new Question record - # points back at its predecessor instead of reopening it. - previous_question_id = Column(Integer, ForeignKey("questions.id"), nullable=True) - - # Primary vs. secondary unresolved_question within its originating situation. - # Secondary questions are tracked but rendered quieter in the brief. - is_primary = Column(Boolean, default=True, nullable=False) - - # Review interval, e.g. "7d", "90d", "1y" (2026-08-27, backlog task 011). - # A cadence is NOT a deadline: it says how often this question is worth - # re-examining, while a Prediction's due_by says when a specific claim - # resolves. A question reviewed quarterly can hold a claim resolving in - # three weeks. - # - # Nullable, and null means "not on any review schedule". Every question - # that predates this column is model-emergent -- the graph noticed it in - # coverage, the operator never put it on an agenda -- so `forecast --due` - # correctly never surfaces it. A cadence is the operator's read on how fast - # a subject moves, so only the operator can set one, via `questions add`. - cadence = Column(String(20), nullable=True) - - # When the operator last reviewed this question. Stamped by `forecast --due` - # whether or not anything moved: a quiet question that reappears every day - # trains the operator to skim. Null means never reviewed, in which case - # first_asked_at is the baseline the interval counts from. - last_reviewed_at = Column(DateTime, nullable=True) - - situation_links = relationship("QuestionSituation", back_populates="question") - - __table_args__ = ( - Index("idx_question_status", "status"), - Index("idx_question_normalized", "normalized_text"), - Index("idx_question_first_asked", "first_asked_at"), - ) - - -class QuestionSituation(Base): - """ - Join row: a Question appeared in a specific situation within a synthesis. - - A Question accumulates QuestionSituation rows over time; the count is the - question's "run number" when surfaced in the brief. - """ - - __tablename__ = "question_situations" - - id = Column(Integer, primary_key=True) - question_id = Column(Integer, ForeignKey("questions.id"), nullable=False) - synthesis_id = Column(Integer, ForeignKey("narrative_syntheses.id"), nullable=False) - situation_index = Column(Integer, nullable=False) - observed_at = Column(DateTime, default=utcnow, nullable=False) - - question = relationship("Question", back_populates="situation_links") - - __table_args__ = ( - Index("idx_qs_question", "question_id"), - Index("idx_qs_synthesis", "synthesis_id"), - UniqueConstraint( - "question_id", "synthesis_id", "situation_index", name="_question_situation_uc" - ), - ) - - -# ============================================================================ -# Predictions Ledger -# Falsifiable observables the synthesis committed to, keyed to Questions. -# A pre-synthesis check pass grades open predictions against fresh coverage, -# which makes the tool's own forward-looking statements auditable over time. -# ============================================================================ - - -PREDICTION_STATUS_OPEN = "open" -PREDICTION_STATUS_TRIGGERED = "triggered" -PREDICTION_STATUS_CONTRADICTED = "contradicted" -PREDICTION_STATUS_EXPIRED = "expired" - -# Open predictions older than this are expired by the check pass; coverage -# that far out is unlikely to still bear on the original observable. -PREDICTION_EXPIRY_DAYS = 90 - -# Who staked the claim (2026-08-27, backlog task 011). The calibration figure -# counts operator predictions only; model predictions stay in the ledger as -# prompts -- suggestions about what is worth holding an opinion on -- and are -# reported under their own heading, never mixed in. -PREDICTION_AUTHOR_OPERATOR = "operator" -PREDICTION_AUTHOR_MODEL = "model" - -# What the operator judged when resolving. Recorded alongside status so the -# verdict survives independently of the status vocabulary the check pass uses. -PREDICTION_OUTCOME_YES = "yes" -PREDICTION_OUTCOME_NO = "no" - - -class Prediction(Base): - """ - A falsifiable observable the synthesis flagged as worth watching. - - Each what_to_watch entry from a situation becomes a Prediction keyed to - that situation's primary Question. The check pass that runs before each - synthesis grades open predictions: triggered (the observable appeared), - contradicted (coverage explicitly went the other way), or expired (aged - out without resolution). - """ - - __tablename__ = "predictions" - - id = Column(Integer, primary_key=True) - question_id = Column(Integer, ForeignKey("questions.id"), nullable=False) - observable_text = Column(Text, nullable=False) - trigger_condition = Column(Text, nullable=False) - - made_at = Column(DateTime, default=utcnow, nullable=False) - # Nullable since 2026-08-27 (backlog task 011): an operator stakes a claim - # from the CLI, with no synthesis behind it. Model predictions still carry - # the synthesis that produced them. - made_in_synthesis_id = Column(Integer, ForeignKey("narrative_syntheses.id"), nullable=True) - - # operator | model. Not nullable, because "who said this" is the one fact - # the calibration figure depends on; every pre-existing row backfills to - # 'model', which is what all 33 of them were. - author = Column(String(20), nullable=False, default=PREDICTION_AUTHOR_MODEL) - - # When this specific claim resolves. Required at entry for an operator - # prediction -- a claim with no resolution date cannot come due and so - # cannot be graded, which is exactly how the ledger ended up with 19 - # predictions that expired unjudged. Null on model predictions, which are - # aged out by PREDICTION_EXPIRY_DAYS instead. - due_by = Column(DateTime, nullable=True) - - # Stated probability, 0.0-1.0. Required at entry for an operator - # prediction and deliberately given no default: an unstated confidence is - # a non-commitment in a different costume. - confidence = Column(Float, nullable=True) - - status = Column(String(20), nullable=False, default=PREDICTION_STATUS_OPEN) - - # yes | no -- the operator's verdict on the claim as written. - outcome = Column(String(20), nullable=True) - - # When the verdict was recorded. Deliberately separate from due_by: - # resolving three months after the due date is itself calibration data. - resolved_at = Column(DateTime, nullable=True) - resolution_note = Column(Text, nullable=True) - - question = relationship("Question") - - __table_args__ = ( - Index("idx_prediction_status", "status"), - Index("idx_prediction_question", "question_id"), - Index("idx_prediction_made_at", "made_at"), - Index("idx_prediction_author", "author"), - Index("idx_prediction_due_by", "due_by"), - ) - - -# ============================================================================ -# Decision Journal -# Standing decisions the user is carrying. Each run routes situation evidence -# into the factors the user cares about, turning daily coverage into an -# accumulating record per decision rather than a one-off briefing. -# ============================================================================ - - -DECISION_STATUS_OPEN = "open" -DECISION_STATUS_DECIDED = "decided" -DECISION_STATUS_DEFERRED = "deferred" - -# How a piece of evidence moved a factor. -EVIDENCE_DIRECTION_SUPPORTS = "supports" -EVIDENCE_DIRECTION_COMPLICATES = "complicates" -EVIDENCE_DIRECTION_NEUTRAL = "neutral" - - -class Decision(Base): - """A standing decision the user is carrying, seeded from the profile.""" - - __tablename__ = "decisions" - - id = Column(Integer, primary_key=True) - name = Column(String(300), nullable=False) - decision_type = Column(String(50)) # career / housing / education / financial / civic / other - status = Column(String(20), nullable=False, default=DECISION_STATUS_OPEN) - created_at = Column(DateTime, default=utcnow, nullable=False) - decided_at = Column(DateTime, nullable=True) - notes = Column(Text, nullable=True) - - factors = relationship("DecisionFactor", back_populates="decision") - evidence = relationship("DecisionEvidence", back_populates="decision") - - __table_args__ = (Index("idx_decision_status", "status"),) - - -class DecisionFactor(Base): - """ - A variable the user is tracking for a decision. ``what_would_update_me`` - is the user's stated rule for what evidence would change their read -- - it is what makes routing tractable, since the router matches situations - against these clauses. - """ - - __tablename__ = "decision_factors" - - id = Column(Integer, primary_key=True) - decision_id = Column(Integer, ForeignKey("decisions.id"), nullable=False) - name = Column(String(300), nullable=False) - what_would_update_me = Column(Text, nullable=True) - current_state_note = Column(Text, nullable=True) - - decision = relationship("Decision", back_populates="factors") - evidence = relationship("DecisionEvidence", back_populates="factor") - - __table_args__ = (Index("idx_decision_factor_decision", "decision_id"),) - - -class DecisionEvidence(Base): - """ - One run's routing result: a situation contained evidence bearing on a - decision factor. This is the only place situation-to-decision routing is - recorded, so the chain from coverage to decision stays inspectable. - """ - - __tablename__ = "decision_evidence" - - id = Column(Integer, primary_key=True) - decision_id = Column(Integer, ForeignKey("decisions.id"), nullable=False) - factor_id = Column(Integer, ForeignKey("decision_factors.id"), nullable=False) - synthesis_id = Column(Integer, ForeignKey("narrative_syntheses.id"), nullable=False) - question_id = Column(Integer, ForeignKey("questions.id"), nullable=True) - - situation_excerpt = Column(Text, nullable=False) - direction = Column(String(20), nullable=False, default=EVIDENCE_DIRECTION_NEUTRAL) - epistemic_status = Column(String(30)) # reported_fact/single_source/consensus/speculation - observed_at = Column(DateTime, default=utcnow, nullable=False) - - decision = relationship("Decision", back_populates="evidence") - factor = relationship("DecisionFactor", back_populates="evidence") - - __table_args__ = ( - Index("idx_decision_evidence_decision", "decision_id"), - Index("idx_decision_evidence_factor", "factor_id"), - Index("idx_decision_evidence_synthesis", "synthesis_id"), - ) - - -# ============================================================================ -# Beats -# A beat is a subject with its own sources, as opposed to the person-shaped -# user profile. These two tables are purely additive (added 2026-08-26 for -# backlog task 004): no existing table gained a beat_id column, because a -# run's beat membership is recorded here and the graph's beat scope is derived -# from it. See docs/CONCEPTS.md, "Beats", for the reasoning. -# ============================================================================ - - -BEAT_RUN_STATUS_STARTED = "started" -BEAT_RUN_STATUS_COMPLETED = "completed" -BEAT_RUN_STATUS_FAILED = "failed" - - -class Beat(Base): - """ - A subject the user runs briefs for, mirroring config/beats/.json. - - The config file is authoritative for sources; this row exists so runs can - be attributed to a beat and so the beat's identity survives a config edit. - """ - - __tablename__ = "beats" - - id = Column(Integer, primary_key=True) - name = Column(String(200), nullable=False, unique=True) - description = Column(Text) - config_path = Column(String(500)) - created_at = Column(DateTime, default=utcnow, nullable=False) - updated_at = Column(DateTime, default=utcnow, onupdate=utcnow) - - runs = relationship("BeatRun", back_populates="beat") - - __table_args__ = (Index("idx_beat_name", "name"),) - - -class BeatRun(Base): - """ - One brief run attributed to a beat. - - This is the join that scopes the graph: a Question or Prediction belongs to - a beat when the synthesis it appeared in has a BeatRun row for that beat, - and belongs to the default (person) scope when its syntheses have none. - """ - - __tablename__ = "beat_runs" - - id = Column(Integer, primary_key=True) - beat_id = Column(Integer, ForeignKey("beats.id"), nullable=False) - analysis_run_id = Column(Integer, ForeignKey("analysis_runs.id"), nullable=True) - synthesis_id = Column(Integer, ForeignKey("narrative_syntheses.id"), nullable=True) - - status = Column(String(20), nullable=False, default=BEAT_RUN_STATUS_COMPLETED) - started_at = Column(DateTime, default=utcnow, nullable=False) - completed_at = Column(DateTime, nullable=True) - articles_analyzed = Column(Integer) - feeds_resolved = Column(Integer) # How many feeds the beat selected at run time - - beat = relationship("Beat", back_populates="runs") - - __table_args__ = ( - Index("idx_beat_run_beat", "beat_id"), - Index("idx_beat_run_synthesis", "synthesis_id"), - Index("idx_beat_run_started_at", "started_at"), - ) - - -class BeatStandingQuestion(Base): - """ - A Question a beat *declared* it is watching, rather than one coverage raised. - - Added 2026-08-26 for backlog task 007. This is a beat table, in the same - sense beat_runs is: the graph tables still carry no beat_id column, and a - beat's scope is still derived from a join rather than stamped on a row. - The join is needed because a declared question can exist before -- and - without ever -- appearing in a synthesis, so beat_runs alone cannot place - it. Without this row such a question would fall into the default scope and - collide with the person brief's ledger, which is exactly what the - derivation rule exists to prevent. - - ``declared_text`` is the config file's wording, kept verbatim so an edit to - the beat file is visible as a difference rather than silently adopted; - ``questions.text`` is what the graph shows. - """ - - __tablename__ = "beat_standing_questions" - - id = Column(Integer, primary_key=True) - beat_id = Column(Integer, ForeignKey("beats.id"), nullable=False) - question_id = Column(Integer, ForeignKey("questions.id"), nullable=False) - - declared_text = Column(Text, nullable=False) - normalized_text = Column(Text, nullable=False) - declared_at = Column(DateTime, default=utcnow, nullable=False) - - question = relationship("Question") - - __table_args__ = ( - Index("idx_bsq_beat", "beat_id"), - Index("idx_bsq_question", "question_id"), - UniqueConstraint("beat_id", "normalized_text", name="_beat_standing_question_uc"), - ) - - -# ============================================================================ -# Institutional activity -# What a beat's declared institutions did this run, against what they usually -# do. Added 2026-08-26 for backlog task 006. -# -# The hard boundary, restated in the schema itself: `kind` admits `org`, -# `program` and `document_type` and nothing else. There is no person kind and -# no persons table, so there is no row a per-individual activity ledger could -# be assembled from. A named individual may appear inside a rendered situation -# where the source document names a signatory -- that is an attribute of a -# document and expires with it -- but never as a row here, because a row here -# accumulates across runs into a file on someone. See -# backlog/006-institutional-activity.md and docs/CONCEPTS.md. -# ============================================================================ - - -ENTITY_KIND_ORG = "org" -ENTITY_KIND_PROGRAM = "program" -ENTITY_KIND_DOCUMENT_TYPE = "document_type" - -# The closed set. Enforced at the loader (src/config/beats.py rejects a -# `coverage.people` key) and again on the write path in -# src/context/institutional_activity.py, which refuses an unknown kind rather -# than storing it. -ENTITY_KINDS = (ENTITY_KIND_ORG, ENTITY_KIND_PROGRAM, ENTITY_KIND_DOCUMENT_TYPE) - - -class BeatEntity(Base): - """ - One institution a beat tracks, mirroring an entry in its `coverage` block. - - The config file stays authoritative for which entities exist and what their - aliases are; this row exists so mentions have something stable to key off - across runs and so an entity keeps its history after a config edit. - """ - - __tablename__ = "beat_entities" - - id = Column(Integer, primary_key=True) - beat_id = Column(Integer, ForeignKey("beats.id"), nullable=False) - - # One of ENTITY_KINDS. Never a person. - kind = Column(String(20), nullable=False) - name = Column(String(200), nullable=False) # canonical form, as configured - aliases = Column(JSON, default=list) # other surface forms of the same institution - - created_at = Column(DateTime, default=utcnow, nullable=False) - updated_at = Column(DateTime, default=utcnow, onupdate=utcnow) - - beat = relationship("Beat") - mentions = relationship("EntityMention", back_populates="entity") - - __table_args__ = ( - Index("idx_beat_entity_beat", "beat_id"), - UniqueConstraint("beat_id", "kind", "name", name="_beat_entity_uc"), - ) - - -class EntityMention(Base): - """ - How many of one run's items mentioned one entity. - - One row per entity per recorded run, **including the zeroes**: a run in - which an office said nothing is an observation about that office, and - without the zero row the trailing average would be an average over only - the days something happened, which always reads as "normal". - - ``item_count`` counts *items*, not occurrences -- an article that names - CISA nine times is one item. Counting repetitions would let a verbose - outlet impersonate institutional activity. - """ - - __tablename__ = "entity_mentions" - - id = Column(Integer, primary_key=True) - entity_id = Column(Integer, ForeignKey("beat_entities.id"), nullable=False) - beat_run_id = Column(Integer, ForeignKey("beat_runs.id"), nullable=True) - synthesis_id = Column(Integer, ForeignKey("narrative_syntheses.id"), nullable=True) - - item_count = Column(Integer, nullable=False, default=0) - items_scanned = Column(Integer) # denominator: how many items this run looked at - observed_at = Column(DateTime, default=utcnow, nullable=False) - - entity = relationship("BeatEntity", back_populates="mentions") - - __table_args__ = ( - Index("idx_entity_mention_entity", "entity_id"), - Index("idx_entity_mention_run", "beat_run_id"), - Index("idx_entity_mention_observed", "observed_at"), - UniqueConstraint("entity_id", "beat_run_id", name="_entity_mention_run_uc"), + # Both composite indexes were added for briefing-era query paths that + # task 012 deleted (curator.py and content_engine.py). They are kept: + # "recent unfiltered articles" is the shape every tier reads, and + # dropping an index is a separate, measurable decision. + Index("idx_articles_filtered_fetched", "filtered", "fetched_at"), + Index("idx_articles_filtered_published", "filtered", "published_date"), ) diff --git a/src/llm/__init__.py b/src/llm/__init__.py new file mode 100644 index 0000000..d692158 --- /dev/null +++ b/src/llm/__init__.py @@ -0,0 +1,12 @@ +""" +The Claude API boundary. + +One client, one response parser. Ported out of the deleted ``src/context/`` +package by backlog task 012 for Tier 2 adjudication; the synthesis product that +used to call it is gone, the call path is not. +""" + +from ._json import parse_claude_json +from .claude_client import ClaudeClient + +__all__ = ["ClaudeClient", "parse_claude_json"] diff --git a/src/context/_json.py b/src/llm/_json.py similarity index 66% rename from src/context/_json.py rename to src/llm/_json.py index 63d2763..e74d01f 100644 --- a/src/context/_json.py +++ b/src/llm/_json.py @@ -1,4 +1,10 @@ -"""Shared JSON parser for Claude responses across the context subsystem.""" +"""Shared JSON parser for Claude responses. + +Moved here from ``src/context/`` with :mod:`src.llm.claude_client` by backlog +task 012. Every caller it had was a briefing-era adjudicator and all of them +were deleted; it stays because a fenced-JSON reply is a property of the API, +not of the product that was reading it, and Tier 2 will parse the same shape. +""" import json import logging diff --git a/src/context/claude_client.py b/src/llm/claude_client.py similarity index 95% rename from src/context/claude_client.py rename to src/llm/claude_client.py index 7ce6aad..7ab03ae 100644 --- a/src/context/claude_client.py +++ b/src/llm/claude_client.py @@ -1,6 +1,11 @@ """ -Minimal Claude API Client -Simple wrapper focused on context-driven analysis +Minimal Claude API client. + +Moved here from ``src/context/`` by backlog task 012 for Tier 2 adjudication. +The synthesis product that used to call it is deleted; this file is not, +because it encodes two outages that cost live runs and that a rewrite from a +blank file would reintroduce -- see :data:`DEFAULT_MODEL` and +:func:`_response_text` below. """ import logging @@ -149,7 +154,7 @@ async def analyze_with_context( Analyze using curated context Args: - context: Curated context dictionary from ContextCurator + context: Curated context dictionary supplied by the caller task: Task description/question effort: Reasoning depth diff --git a/src/matching/__init__.py b/src/matching/__init__.py new file mode 100644 index 0000000..f7562c7 --- /dev/null +++ b/src/matching/__init__.py @@ -0,0 +1,19 @@ +""" +Deterministic text matching. + +No model is involved anywhere in this package. A match is a word-boundary regex +hit, which makes every count reproducible from the same articles forever and +makes a wrong count debuggable by reading a regex rather than by re-running a +prompt. + +Ported out of the deleted ``src/context/`` package by backlog task 012: + +* :mod:`~src.matching.entity_matcher` -- alias matching for Tier 1 routing. +* :mod:`~src.matching.coverage_probe` -- "did anything match this in N days", + which is the staleness check task 018 needs. +* :mod:`~src.matching.terms` -- the value objects both consume. +""" + +from .terms import CoverageEntity, CoverageProbe + +__all__ = ["CoverageEntity", "CoverageProbe"] diff --git a/src/context/coverage_probe.py b/src/matching/coverage_probe.py similarity index 79% rename from src/context/coverage_probe.py rename to src/matching/coverage_probe.py index 8758ea1..26ae8ba 100644 --- a/src/context/coverage_probe.py +++ b/src/matching/coverage_probe.py @@ -1,20 +1,20 @@ """ -Testing a beat against events that actually happened. +Testing a set of sources against events that actually happened. -Task 005 asked whether a beat could reach its domain and answered with document -counts: 469 Federal Register documents narrowed to 24, 38 retrieved in seven -days. Both real, both green, and neither able to detect that the beat was -structurally unable to see a personnel change -- because a personnel change is -not a document. The beat's first live brief then missed the reinstatement of the -FedRAMP director, the largest event in its declared domain that week. +Task 005 asked whether a source list could reach its domain and answered with +document counts: 469 Federal Register documents narrowed to 24, 38 retrieved in +seven days. Both real, both green, and neither able to detect that the sources +were structurally unable to see a personnel change -- because a personnel change +is not a document. The first live run then missed the reinstatement of the +FedRAMP director, the largest event in the declared domain that week. **An article count measures whether ingestion is running. It does not measure whether ingestion reaches the domain.** A coverage probe measures the second thing, by the only method that cannot be satisfied by ingesting more of what the -beat already reads: +sources already carry: Name something that actually happened in this domain, then check whether the - beat can see it. + sources can see it. Nothing here calls a model. A probe either matched a stored article or it did not, and the answer is reproducible from the same rows forever and debuggable by @@ -27,7 +27,7 @@ ``nist`` -- "administration", "minister", "Afghanistan", "communist", "sinister", "columnist" -- and exactly 4 are about NIST. A substring matcher would report 99.4% false coverage on that one term alone. Terms are anchored - at word boundaries by :mod:`src.context.entity_matcher`, which is the same + at word boundaries by :mod:`src.matching.entity_matcher`, which is the same code path coverage entities use. * **Stems are marked, never inferred.** ``reinstat*`` matches "reinstated" and "reinstatement"; ``reinstat`` matches neither. Inferring the widening would @@ -35,11 +35,14 @@ direction that manufactures false confidence. * **A window with no articles in it is not a failure.** A probe whose window predates the corpus, or falls in a gap, is ``INCONCLUSIVE`` -- it says nothing - about the beat's reach, because there was nothing to reach with. It stays in + about the sources' reach, because there was nothing to reach with. It stays in the count regardless, since a probe set that quietly shrinks to the events still in retention is a green light that means nothing. -Added 2026-08-27 for backlog task 010. +Added 2026-08-27 for backlog task 010. Moved here from ``src/context/`` by task +012 and decoupled from the deleted ``BeatConfig``: the caller now supplies the +probes and the feed URLs directly. "Did anything match this in N days" is the +staleness check task 018 needs, and this is already that engine. """ from __future__ import annotations @@ -51,9 +54,9 @@ from sqlalchemy.orm import Session -from ..config.beats import BeatConfig, CoverageProbe from ..database.models import Article, RSSFeed from .entity_matcher import is_shouted, term_pattern +from .terms import CoverageProbe __all__ = [ "STATUS_INCONCLUSIVE", @@ -67,9 +70,9 @@ "run_coverage_probes", ] -# A probe matched at least one article the beat's own sources carried. +# A probe matched at least one article the selected sources carried. STATUS_MATCHED = "MATCHED" -# The window held articles from the beat's sources and none of them matched. +# The window held articles from the selected sources and none of them matched. # This is the finding the feature exists to produce. STATUS_UNMATCHED = "UNMATCHED" # The window held nothing to match against, so the probe answers nothing. @@ -80,8 +83,8 @@ # essentially all of the collision filtering. STEM_MARKER = "*" -# How many matching articles to keep per probe. The question is "can the beat -# see this", not "how loudly", so a handful of examples is the whole evidence a +# How many matching articles to keep per probe. The question is "can these +# sources see this", not "how loudly", so a handful of examples is the evidence a # reader needs and an unbounded list would bury the verdict. DEFAULT_MATCH_LIMIT = 5 @@ -168,7 +171,7 @@ class ProbeMatch: title: str feed_name: str published_date: datetime | None - in_beat_sources: bool + in_selected_sources: bool @property def published_day(self) -> str: @@ -180,11 +183,11 @@ class ProbeResult: """ What one probe found. - ``elsewhere`` is only populated for a probe the beat's own sources missed: - it names articles from feeds *outside* this beat that would have satisfied - it. That distinction is the actionable part. "Nobody in the corpus carried - it" and "three feeds you do not subscribe to carried it" are the same - ``UNMATCHED`` verdict and completely different repairs. + ``elsewhere`` is only populated for a probe the selected sources missed: it + names articles from feeds *outside* that selection which would have + satisfied it. That distinction is the actionable part. "Nobody in the corpus + carried it" and "three feeds you do not subscribe to carried it" are the + same ``UNMATCHED`` verdict and completely different repairs. """ probe: CoverageProbe @@ -211,16 +214,19 @@ def conclusive(self) -> bool: @dataclass(frozen=True) class CoverageReport: """ - Every probe's result for one beat, plus the corpus facts that framed them. + Every probe's result for one subject, plus the corpus facts that framed them. - ``total`` is the number of probes the beat *declared*, and the passed, + ``total`` is the number of probes the caller *supplied*, and the passed, unmatched and inconclusive counts always sum to it. The invariant is the point: an inconclusive probe is subtracted from neither side, so a probe set rotting out of retention shows up as a shrinking denominator of *answered* questions rather than as a clean sweep of the ones that remain. + + ``subject`` is a free-text label for whatever the probe set belongs to. It + was ``beat_name`` until task 012 deleted beats; nothing here interprets it. """ - beat_name: str + subject: str results: tuple[ProbeResult, ...] feed_names: tuple[str, ...] earliest: datetime | None @@ -252,7 +258,7 @@ def article_text(article: Article) -> str: The text of one stored article that probe matching reads. Title, summary and normalized body. ``author`` is excluded, as it is in - :func:`src.context.entity_matcher.item_text`: a byline is a person, and a + :func:`src.matching.entity_matcher.item_text`: a byline is a person, and a probe is about whether a source carried an event, never about who wrote it. """ parts = [ @@ -269,13 +275,13 @@ def _window_bounds(probe: CoverageProbe) -> tuple[datetime, datetime]: return datetime.combine(start, time.min), datetime.combine(end, time.max) -def _to_match(article: Article, feed_name: str | None, in_beat_sources: bool) -> ProbeMatch: +def _to_match(article: Article, feed_name: str | None, in_selected_sources: bool) -> ProbeMatch: return ProbeMatch( article_id=article.id, title=article.title or "(untitled)", feed_name=feed_name or "(unattributed source)", published_date=article.published_date, - in_beat_sources=in_beat_sources, + in_selected_sources=in_selected_sources, ) @@ -320,7 +326,7 @@ class _Evaluation: def _evaluate( compiled: CompiledProbe, rows: Iterable[tuple[Article, str | None]], - in_beat_sources: bool, + in_selected_sources: bool, limit: int, ) -> _Evaluation: """ @@ -347,7 +353,7 @@ def _evaluate( if not absent and not unsatisfied: match_count += 1 if len(matches) < limit: - matches.append(_to_match(article, feed_name, in_beat_sources)) + matches.append(_to_match(article, feed_name, in_selected_sources)) continue score = len(absent) + len(unsatisfied) if score < best_score: @@ -358,25 +364,29 @@ def _evaluate( def run_coverage_probes( session: Session, - beat: BeatConfig, - feed_urls: Sequence[str] | None = None, + probes: Sequence[CoverageProbe], + feed_urls: Sequence[str], + *, + subject: str = "", match_limit: int = DEFAULT_MATCH_LIMIT, ) -> CoverageReport: """ - Run every probe a beat declares against the corpus, read-only. + Run every supplied probe against the corpus, read-only. + + The verdict is scoped to ``feed_urls``: a probe passes when one of *those* + feeds carried the event. An unmatched probe is then re-run across the whole + corpus, and any hit is reported as ``elsewhere`` -- evidence that the event + was reachable and the selected source list is what missed it. - The verdict is scoped to the beat's own sources: a probe passes when a feed - *this beat subscribes to* carried the event. An unmatched probe is then - re-run across the whole corpus, and any hit is reported as ``elsewhere`` -- - evidence that the event was reachable and this beat's source list is what - missed it. + An empty ``probes`` returns an empty report. That is not a pass, and the + caller is expected to say so: an undeclared probe set is the state the old + beat was already in when it missed the FedRAMP reinstatement. - A beat with no probes returns an empty report. That is not a pass, and the - caller is expected to say so: an undeclared probe set is the state the beat - was already in when it missed the FedRAMP reinstatement. + Before task 012 this took a ``BeatConfig`` and called ``resolve_feeds()`` + itself. Beats are gone; the caller now names the scope, which is also what + a staleness check wants -- it already knows which feeds it is asking about. """ - feeds = beat.resolve_feeds() - urls = list(feed_urls) if feed_urls is not None else [feed.url for feed in feeds] + urls = list(feed_urls) feed_rows = session.query(RSSFeed).filter(RSSFeed.url.in_(urls)).all() if urls else [] feed_ids = [row.id for row in feed_rows] feed_names = tuple(sorted(row.name for row in feed_rows)) @@ -384,7 +394,7 @@ def run_coverage_probes( earliest, latest = _corpus_bounds(session, feed_ids) results: list[ProbeResult] = [] - for probe in beat.coverage_probes: + for probe in probes: compiled = compile_probe(probe) rows = _rows_in_window(session, probe, feed_ids) if feed_ids else [] found = _evaluate(compiled, rows, True, match_limit) @@ -431,7 +441,7 @@ def run_coverage_probes( ) return CoverageReport( - beat_name=beat.name, + subject=subject, results=tuple(results), feed_names=feed_names, earliest=earliest, @@ -442,7 +452,7 @@ def run_coverage_probes( def _corpus_bounds( session: Session, feed_ids: Sequence[int] ) -> tuple[datetime | None, datetime | None]: - """Oldest and newest dated article carried by the beat's own sources.""" + """Oldest and newest dated article carried by the selected sources.""" if not feed_ids: return None, None query = session.query(Article.published_date).filter( @@ -468,21 +478,21 @@ def _inconclusive_reason( that is worth knowing on its own. """ if not has_feeds: - return "this beat's sources have no rows in the corpus, so nothing was searched" + return "these sources have no rows in the corpus, so nothing was searched" if earliest is None: - return "this beat's sources have stored no dated articles, so nothing was searched" + return "these sources have stored no dated articles, so nothing was searched" start, end = probe.window if end < earliest.date(): return ( - f"window ends {end.isoformat()}, before the beat's oldest stored article " + f"window ends {end.isoformat()}, before the oldest stored article " f"({earliest.date().isoformat()}) -- the event predates the corpus" ) if latest is not None and start > latest.date(): return ( - f"window starts {start.isoformat()}, after the beat's newest stored article " + f"window starts {start.isoformat()}, after the newest stored article " f"({latest.date().isoformat()}) -- the corpus has not reached the event yet" ) return ( - "the beat's sources stored no dated articles anywhere in this window -- " - "a gap in the corpus, not a gap in the beat" + "these sources stored no dated articles anywhere in this window -- " + "a gap in the corpus, not a gap in the source list" ) diff --git a/src/context/entity_matcher.py b/src/matching/entity_matcher.py similarity index 93% rename from src/context/entity_matcher.py rename to src/matching/entity_matcher.py index c92b819..8bced2b 100644 --- a/src/context/entity_matcher.py +++ b/src/matching/entity_matcher.py @@ -20,9 +20,12 @@ case-varied in prose and are long enough not to collide. Nothing in this module knows what a person is, because nothing it can be -handed is one: it takes :class:`~src.config.beats.CoverageEntity` values, whose -``kind`` is restricted to ``org`` / ``program`` / ``document_type`` by the -loader. +handed is one: it takes :class:`~src.matching.terms.CoverageEntity` values, +whose ``kind`` is restricted to ``org`` / ``program`` / ``document_type``. + +Moved here from ``src/context/`` by backlog task 012. The beat whose coverage +it used to count is gone; the boundary and case rules below are what Tier 1 +routing needs and are not re-derivable by reading the code that calls them. """ from __future__ import annotations @@ -32,7 +35,7 @@ from dataclasses import dataclass from typing import Any -from ..config.beats import CoverageEntity +from .terms import CoverageEntity __all__ = [ "LEFT_BOUNDARY", diff --git a/src/matching/terms.py b/src/matching/terms.py new file mode 100644 index 0000000..a5e6e3c --- /dev/null +++ b/src/matching/terms.py @@ -0,0 +1,142 @@ +""" +What a matcher is pointed at: entities and probes. + +These two value objects were lifted verbatim from ``src/config/beats.py`` when +the briefing product was deleted (backlog task 012). The *beat* they used to +belong to is gone -- a beat was a subject with its own source list, and the new +shape tracks one operator's decisions instead. What survives is the pair of +descriptions the deterministic matchers consume: + +* :class:`CoverageEntity` -- an institution and its surface forms, read by + :mod:`src.matching.entity_matcher` (Tier 1 routing). +* :class:`CoverageProbe` -- one thing that actually happened plus the words any + report of it would carry, read by :mod:`src.matching.coverage_probe` (the + staleness check task 018 needs). + +They carry no loader. The JSON beat-file parsing and validation that used to +build them died with ``src/config/beats.py``; whatever declares an entity or a +probe in the new shape constructs these directly. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date, timedelta + +__all__ = [ + "COVERAGE_KINDS", + "DEFAULT_PROBE_WINDOW_DAYS", + "ENTITY_KINDS", + "MIN_PROBE_EVIDENCE", + "CoverageEntity", + "CoverageProbe", +] + +# How far either side of a probe's date a report of it is still a report of it. +# Symmetric because coverage runs both ways: a rule is trailed before it lands +# and analysed for a week after. Fourteen days is the interval the FedRAMP miss +# was measured over -- "3 incidental mentions and none within two weeks". +DEFAULT_PROBE_WINDOW_DAYS = 14 + +# The least evidence a probe may rest on. A probe is a claim that a match means +# the matcher saw a specific event, and one bare term cannot carry that claim: +# `FedRAMP` alone is matched by an AWS region-launch post. Two independent +# pieces of evidence -- two required terms, or one required term plus one +# `any_of` group -- is the floor. The beat-file loader used to enforce this at +# parse time; with the loader gone it stands here as the documented floor for +# whatever constructs probes next. +MIN_PROBE_EVIDENCE = 2 + +# The only three things coverage may track, and the plural config key that used +# to declare each. The mapping is closed on purpose: any other key -- `people`, +# `officials`, `staff`, anything -- was a validation error rather than a +# silently ignored block, so the boundary cannot be reintroduced by convention. +COVERAGE_KINDS: dict[str, str] = { + "orgs": "org", + "programs": "program", + "document_types": "document_type", +} +ENTITY_KINDS = frozenset(COVERAGE_KINDS.values()) + + +@dataclass(frozen=True) +class CoverageEntity: + """ + One institution being tracked: an organization, a program, or a type of + document. + + ``kind`` is one of :data:`ENTITY_KINDS`. There is no person kind, and the + absence is the point -- personnel rotate while offices persist, so a name + goes dark on reassignment and the silence reads as inactivity, which is a + wrong answer that looks like a real one. ``name`` is the canonical form + used everywhere the entity is displayed or stored; ``aliases`` are the + other surface forms that count as the same entity. + """ + + kind: str + name: str + aliases: tuple[str, ...] = () + + @property + def key(self) -> str: + """Stable identity of this entity: ``kind:name``.""" + return f"{self.kind}:{self.name}" + + @property + def terms(self) -> tuple[str, ...]: + """ + Every surface form that counts as this entity, canonical name first. + + Deduplicated but order-preserving, so matching is deterministic + regardless of how the source repeated itself. + """ + seen: dict[str, None] = {} + for term in (self.name, *self.aliases): + seen.setdefault(term, None) + return tuple(seen) + + +@dataclass(frozen=True) +class CoverageProbe: + """ + One thing that actually happened, used to test whether coverage can see it. + + A probe is not a search. It is a claim of the form "an event of this + description occurred on this date, and any report of it would carry these + words" -- which makes an unmatched probe a statement about the source list + rather than about the phrasing of a query. + + ``terms`` must **all** appear in the same article; each group in ``any_of`` + contributes one alternative that must appear. The two levels exist because + a single distinctive term is too weak to be evidence and a whole headline is + too brittle to survive a second outlet's phrasing. + + A term ending in ``*`` is a stem: ``reinstat*`` matches "reinstated" and + "reinstatement". Without the marker a term matches whole words only. The + marker is explicit rather than implied so that the widening is visible to + whoever has to trust the result -- see :mod:`src.matching.coverage_probe` + for the matching rules themselves. + """ + + date: date + what: str + terms: tuple[str, ...] + any_of: tuple[tuple[str, ...], ...] = () + window_days: int = DEFAULT_PROBE_WINDOW_DAYS + + @property + def window(self) -> tuple[date, date]: + """The inclusive date range in which a report of this event counts.""" + span = timedelta(days=self.window_days) + return self.date - span, self.date + span + + @property + def evidence_count(self) -> int: + """How many independent things this probe requires of an article.""" + return len(self.terms) + len(self.any_of) + + def describe(self) -> str: + """The probe's requirement, written the way it was declared.""" + parts = [" AND ".join(self.terms)] if self.terms else [] + parts.extend("(" + " OR ".join(group) + ")" for group in self.any_of) + return " AND ".join(parts) diff --git a/src/pipeline/__init__.py b/src/pipeline/__init__.py deleted file mode 100644 index 3be37e3..0000000 --- a/src/pipeline/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -InsightWeaver Data Pipeline -Orchestrates RSS fetching, processing, and analysis -""" diff --git a/src/pipeline/orchestrator.py b/src/pipeline/orchestrator.py deleted file mode 100644 index 345bc93..0000000 --- a/src/pipeline/orchestrator.py +++ /dev/null @@ -1,383 +0,0 @@ -""" -Pipeline Orchestrator -Coordinates RSS fetching, deduplication, and prioritization in sequence -""" - -import asyncio -import logging -from datetime import UTC, datetime -from typing import Any - -from src.config.beats import BeatConfig -from src.config.settings import settings -from src.context.synthesizer import NarrativeSynthesizer -from src.processors.content_filter import ContentFilter -from src.processors.deduplicator import run_deduplication -from src.rss.parallel_fetcher import fetch_all_active_feeds -from src.sources.runner import run_configured_adapters -from src.utils.profile_loader import get_user_profile -from src.utils.profiler import get_profiler, profile - -logger = logging.getLogger(__name__) - - -class PipelineOrchestrator: - """Orchestrates the complete data pipeline""" - - def __init__( - self, - max_concurrent_feeds: int = 10, - rate_limit: float = 2.0, - dedup_hours: int = 24, - prioritize_hours: int = 48, - prioritize_limit: int | None = None, - topic_filters: dict | None = None, - beat: BeatConfig | None = None, - ): - """ - Initialize pipeline orchestrator - - Args: - max_concurrent_feeds: Max concurrent RSS feed fetches - rate_limit: Requests per second for RSS fetching - dedup_hours: Hours to look back for deduplication - prioritize_hours: Hours to look back for prioritization - prioritize_limit: Max articles to prioritize (None = all) - topic_filters: Optional topic/scope filters for article selection - beat: Optional beat to scope synthesis to. None is the default - person-profile brief and leaves every stage unchanged. - """ - self.max_concurrent_feeds = max_concurrent_feeds - self.rate_limit = rate_limit - self.dedup_hours = dedup_hours - self.prioritize_hours = prioritize_hours - self.prioritize_limit = prioritize_limit - self.topic_filters = topic_filters or {} - self.beat = beat - self.content_filter = None - - def _should_skip_rss_fetch(self, max_age_minutes: int = 60) -> tuple: - """ - Check if RSS fetching should be skipped based on recent article freshness - - Args: - max_age_minutes: Maximum age of articles in minutes to consider fresh - - Returns: - tuple: (should_skip: bool, most_recent_timestamp: Optional[datetime]) - """ - from src.database.connection import get_db - from src.database.models import Article - - try: - with get_db() as session: - most_recent = ( - session.query(Article.fetched_at).order_by(Article.fetched_at.desc()).first() - ) - - if not most_recent or not most_recent[0]: - logger.info("No articles found in database, will fetch RSS feeds") - return False, None - - most_recent_time = most_recent[0] - - # Handle timezone-aware comparison - now = datetime.now(UTC) - if most_recent_time.tzinfo is None: - most_recent_time = most_recent_time.replace(tzinfo=UTC) - - age_minutes = (now - most_recent_time).total_seconds() / 60 - should_skip = age_minutes < max_age_minutes - - if should_skip: - logger.info( - f"Most recent articles are {age_minutes:.1f} minutes old (<{max_age_minutes}m threshold), skipping RSS fetch" - ) - else: - logger.info( - f"Most recent articles are {age_minutes:.1f} minutes old (>{max_age_minutes}m threshold), will fetch RSS feeds" - ) - - return should_skip, most_recent_time - - except Exception as e: - logger.error(f"Error checking article freshness: {e}") - # Fail-safe: always fetch on error - return False, None - - async def run_full_pipeline(self) -> dict[str, Any]: - """ - Run the complete pipeline: - 1. Fetch RSS feeds - 2. Deduplicate articles - 3. Filter content (based on user profile) - 4. Synthesize narrative (if API key configured) - - Returns: - Dictionary with results from each stage - """ - # Start profiling session - profiler = get_profiler() - profiler.start_session() - - results = {"pipeline_started": datetime.now(UTC).isoformat(), "stages": {}} - - try: - # Stage 1: Smart RSS Feed Fetching - with profile("STAGE_1_RSS_FETCH"): - logger.info("Starting Stage 1: RSS Feed Fetching") - - # Check if smart RSS fetch is enabled - smart_fetch_enabled = getattr(settings, "enable_smart_rss_fetch", True) - smart_fetch_threshold = getattr(settings, "smart_rss_fetch_threshold_minutes", 60) - - skip_fetch = False - most_recent = None - - if smart_fetch_enabled: - skip_fetch, most_recent = self._should_skip_rss_fetch(smart_fetch_threshold) - - if skip_fetch: - logger.info(f"Skipping RSS fetch (articles < {smart_fetch_threshold} min old)") - results["stages"]["fetch"] = { - "skipped": True, - "reason": f"Articles less than {smart_fetch_threshold} minutes old", - "most_recent_fetch": most_recent.isoformat() if most_recent else None, - "time_saved_seconds": 60, - } - else: - fetch_results = await self._fetch_feeds() - results["stages"]["fetch"] = fetch_results - logger.info( - f"Stage 1 complete: {fetch_results['successful_feeds']}/{fetch_results['total_feeds']} feeds fetched" - ) - - # Stage 2: Deduplication (only if we got new articles) - with profile("STAGE_2_DEDUP"): - fetch_stage = results["stages"]["fetch"] - if ( - not fetch_stage.get("skipped", False) - and fetch_stage.get("total_articles", 0) > 0 - ): - logger.info("Starting Stage 2: Article Deduplication") - dedup_results = await self._deduplicate_articles() - results["stages"]["deduplication"] = dedup_results - logger.info( - f"Stage 2 complete: {dedup_results['total_duplicates']} duplicates found" - ) - else: - reason = ( - "RSS fetch was skipped" if fetch_stage.get("skipped") else "No new articles" - ) - logger.info(f"Skipping Stage 2: {reason}") - results["stages"]["deduplication"] = {"skipped": True, "reason": reason} - - # Stage 3: Content Filtering (based on user profile) - with profile("STAGE_3_FILTER"): - logger.info("Starting Stage 3: Content Filtering") - filter_results = await self._filter_content() - results["stages"]["filtering"] = filter_results - logger.info( - f"Stage 3 complete: {filter_results['filtered_count']} articles filtered" - ) - - # Stage 4: Narrative Synthesis (only if API key configured) - with profile("STAGE_4_SYNTHESIS"): - if settings.anthropic_api_key: - logger.info("Starting Stage 4: Narrative Synthesis") - synthesis_results = await self._synthesize_narrative() - results["stages"]["synthesis"] = synthesis_results - - articles_synthesized = synthesis_results.get("articles_analyzed", 0) - logger.info( - f"Stage 4 complete: {articles_synthesized} articles synthesized into narrative" - ) - else: - logger.info("Skipping Stage 4: ANTHROPIC_API_KEY not configured") - results["stages"]["synthesis"] = { - "skipped": True, - "reason": "API key not configured", - } - - # Calculate pipeline summary - results["pipeline_completed"] = datetime.now(UTC).isoformat() - results["summary"] = self._generate_summary(results) - - logger.info( - f"Pipeline complete: {results['summary']['total_stages_run']} stages run successfully" - ) - - except Exception as e: - logger.error(f"Pipeline failed: {e}") - results["error"] = str(e) - results["pipeline_failed"] = datetime.now(UTC).isoformat() - raise - - return results - - async def _fetch_feeds(self) -> dict[str, Any]: - """ - Run the ingestion stage: RSS feeds, then every non-RSS source adapter. - - The RSS half is untouched. The adapter half was added 2026-08-26 for - backlog task 005 so the beat can reach sources that publish no usable - RSS. Both halves write the same ``articles`` row shape, so every later - stage is unaware that this stage grew a second half. - """ - results = await fetch_all_active_feeds( - max_concurrent=self.max_concurrent_feeds, rate_limit=self.rate_limit - ) - - adapter_summary = await run_configured_adapters() - if adapter_summary.total_sources: - results["total_feeds"] += adapter_summary.total_sources - results["successful_feeds"] += adapter_summary.successful_sources - results["failed_feeds"] += ( - adapter_summary.total_sources - adapter_summary.successful_sources - ) - results["total_articles"] += adapter_summary.total_articles - results["adapters"] = adapter_summary.as_dict() - # Surfaced all the way to the CLI: an unreachable or newly silent - # source must not be absorbed into a total that still looks fine. - results["source_alerts"] = adapter_summary.alerts - - return results - - async def _deduplicate_articles(self) -> dict[str, Any]: - """Run article deduplication stage""" - # Deduplication is synchronous, wrap in executor - return await asyncio.get_event_loop().run_in_executor( - None, run_deduplication, self.dedup_hours - ) - - async def _filter_content(self) -> dict[str, Any]: - """Run content filtering stage""" - from datetime import timedelta - - from src.database.connection import get_db - from src.database.models import Article - from src.utils import utcnow - - if not self.content_filter: - try: - user_profile = get_user_profile() - self.content_filter = ContentFilter(user_profile) - except FileNotFoundError as e: - logger.error(f"User profile not found: {e}") - return {"skipped": True, "reason": "User profile not found", "error": str(e)} - - with get_db() as session: - cutoff_time = utcnow() - timedelta(hours=self.dedup_hours) - - articles = ( - session.query(Article) - .filter( - Article.created_at >= cutoff_time, - Article.filtered.is_(False), - ) - .all() - ) - - if not articles: - return { - "articles_evaluated": 0, - "filtered_count": 0, - "kept_count": 0, - "filter_rate": 0, - "reasons": {}, - } - - self.content_filter.filter_articles(articles) - stats = self.content_filter.get_filter_stats(articles) - - logger.info( - f"Content filtering: {stats['filtered_count']}/{stats['total_articles']} filtered ({stats['filter_rate']:.1%})" - ) - for reason, count in stats["reasons"].items(): - logger.info(f" - {reason}: {count} articles") - - return stats - - async def _synthesize_narrative(self) -> dict[str, Any]: - """ - Two-pass situation synthesis. - - Pass 1: Cluster articles into topics. - Pass 2: Examined narratives for clusters with 3+ articles, - thin coverage summaries for the rest. - """ - synthesizer = NarrativeSynthesizer(topic_filters=self.topic_filters, beat=self.beat) - return await synthesizer.synthesize(hours=self.prioritize_hours, max_articles=50) - - def _generate_summary(self, results: dict[str, Any]) -> dict[str, Any]: - """Generate pipeline execution summary""" - stages = results.get("stages", {}) - - # Count successful stages - stages_run = len([s for s in stages.values() if not s.get("skipped")]) - - # Extract key metrics - fetch_stage = stages.get("fetch", {}) - dedup_stage = stages.get("deduplication", {}) - filter_stage = stages.get("filtering", {}) - synthesis_stage = stages.get("synthesis", {}) - - summary = { - "total_stages_run": stages_run, - "feeds_processed": fetch_stage.get("total_feeds", 0), - "articles_fetched": fetch_stage.get("total_articles", 0), - "duplicates_removed": dedup_stage.get("total_duplicates", 0), - "articles_filtered": filter_stage.get("filtered_count", 0), - "articles_kept": filter_stage.get("kept_count", 0), - "articles_synthesized": synthesis_stage.get("articles_analyzed", 0), - "narrative_generated": synthesis_stage.get("synthesis_id") is not None, - # Carried into the summary so the CLI cannot render a brief without - # also rendering the fact that a source failed or went silent. - "source_alerts": fetch_stage.get("source_alerts", []), - } - - # Calculate duration - if "pipeline_started" in results and "pipeline_completed" in results: - start = datetime.fromisoformat(results["pipeline_started"]) - end = datetime.fromisoformat(results["pipeline_completed"]) - duration = (end - start).total_seconds() - summary["duration_seconds"] = round(duration, 2) - - return summary - - -async def run_pipeline( - max_concurrent: int = 10, - rate_limit: float = 2.0, - dedup_hours: int = 24, - prioritize_hours: int = 48, - prioritize_limit: int | None = None, - topic_filters: dict | None = None, - beat: BeatConfig | None = None, -) -> dict[str, Any]: - """ - Convenience function to run the complete pipeline - - Args: - max_concurrent: Max concurrent RSS feeds to fetch - rate_limit: Requests per second for RSS fetching - dedup_hours: Hours to look back for deduplication - prioritize_hours: Hours to look back for prioritization - prioritize_limit: Max articles to prioritize (None = all) - topic_filters: Optional topic/scope filters for article selection - beat: Optional beat to scope synthesis to (None = default brief) - - Returns: - Pipeline execution results - """ - orchestrator = PipelineOrchestrator( - max_concurrent_feeds=max_concurrent, - rate_limit=rate_limit, - dedup_hours=dedup_hours, - prioritize_hours=prioritize_hours, - prioritize_limit=prioritize_limit, - topic_filters=topic_filters, - beat=beat, - ) - - return await orchestrator.run_full_pipeline() diff --git a/src/prompts/ANALYSIS_RULES.md b/src/prompts/ANALYSIS_RULES.md deleted file mode 100644 index f397231..0000000 --- a/src/prompts/ANALYSIS_RULES.md +++ /dev/null @@ -1,85 +0,0 @@ -# Analysis Rules - -These rules govern all narrative synthesis output. The goal is that a reader finishes each situation analysis understanding five things: - -1. What is actually happening (not just what is being reported) -2. Who is doing what to whom, and why -3. What the coverage makes easy to see and what it makes hard to see -4. Where the story could go, and what would have to be true for each path -5. What they would need to know to make a good decision about this - -Follow these rules exactly. - -## Epistemic Status Labeling - -Every claim in your output must carry one of these labels: - -- **Reported fact**: Multiple independent sources report this. Cite them. -- **Single-source claim**: Only one source asserts this. Name the source explicitly. -- **Consensus view**: Broad expert or institutional agreement exists. State whose consensus. -- **Speculation**: No direct evidence supports this; it is an inference or prediction. Say so plainly. - -When a claim's status is ambiguous, default to the weaker label. "Single-source claim" is weaker than "reported fact." "Speculation" is weaker than "single-source claim." - -Do not embed epistemic status in hedging language ("some say", "it appears"). Use the labels directly. - -## Narrative Layers and Conflict - -There is never just one narrative. Multiple narrative layers coexist in any situation. Your job is to identify them and map their relationships. - -For each situation: - -1. **Identify the operative narratives** -- not just the dominant frame, but the distinct stories being told about what is happening and why. Each narrative emphasizes different facts, assumes different premises, and implies different outcomes. -2. **Map the fractures** -- where do these narratives conflict? What does one narrative assert that another denies or ignores? Name the specific point of disagreement, not just the fact that disagreement exists. -3. **Look for bridges** -- is there any claim, evidence, or actor that connects conflicting narratives? If a bridge exists, describe it. If no bridge exists, say so -- an unresolved fissure is the most important thing to surface. -4. **Identify which layer is operative for which actors** -- different actors in the same situation may be operating under different narratives. A legislator's public statements may use one narrative while their voting record reflects another. Name these discrepancies when the evidence supports them. - -When narratives conflict, describe the conflict. Do not resolve it unless evidence favors one side. The fissure itself is insight. - -## What the Coverage Makes Hard to See - -Every information source has structural limits on what it can say. These limits are not always about bias in the partisan sense. They arise from: - -- **Institutional alignment**: A source that depends on access to powerful actors will not publish narratives that threaten that access. Name this when you can identify it. -- **Beat constraints**: A reporter covering a specific institution adopts that institution's framing by default. A Pentagon correspondent writes about defense policy in defense terms. Name the beat constraint when it shapes coverage. -- **Cultural tellability**: Some narratives are structurally absent -- not because no one holds the perspective, but because the context in which the coverage is produced makes that perspective difficult to articulate. When you identify this, name it as a structural absence, not a missing opinion. - -Do not fabricate what the absent perspective would say. Name the absence and explain why the information environment produces it. - -## Where the Story Could Go - -For each situation, identify the branching paths: - -- **What would have to be true** for the situation to develop in one direction versus another. Frame these as conditional: "if X, then Y follows." -- **Which narrative layer predicts which outcome** -- conflicting narratives often imply different futures. Name which narrative predicts what. -- **What is the unresolved question** that determines which path the situation takes. This is often the most valuable thing in the entire analysis. - -Do not predict outcomes. Map the conditional structure so the reader can evaluate the paths themselves. - -## Information Gaps - -When your analysis reveals an information gap, flag it explicitly: - -- **What is missing**: The specific information or perspective not present. -- **Why it matters**: How this gap affects the reader's ability to understand the situation or make a decision about it. -- **Why it might be missing**: Is this a data gap (no one has measured it), a coverage gap (sources that cover this beat don't report on it), or a structural absence (the information environment makes this perspective hard to articulate)? -- **Feed recommendation**: If a type of source would fill this gap, name it. This is a feed curation signal, not a search query. - -Gaps are first-class output. A synthesis that honestly maps what it does not know is more useful than one that papers over holes. - -## Uncertainty Expression - -Express uncertainty as a structural feature of the information, not as a rhetorical hedge. - -Instead of: "It's possible that supply chain disruptions could affect pricing." -Write: "Three sources report supply chain disruptions in semiconductor manufacturing. No source in today's coverage quantifies the pricing impact. This is a data gap. If the disruption persists, the pricing impact depends on inventory levels, which no source reports." - -When information is insufficient to draw a conclusion, say what is missing and stop. Do not fill gaps with plausible-sounding inferences. - -## Structural Rules - -- Do not use emotional language or urgency framing ("alarming", "critical", "unprecedented") unless directly quoting a source. If quoting, attribute it. -- Do not rank stories by implied importance through adjective choice. Use the priority structure provided in the synthesis task. -- When two sources contradict each other, present both with attribution. Do not resolve the contradiction unless a third source provides evidence favoring one side. -- Do not anthropomorphize markets, institutions, or abstract systems ("the market fears", "Congress wants"). Describe observable actions and stated positions. -- Do not recommend actions. Map implications: "if X is true, then Y follows for someone in this situation." The reader draws their own conclusions. diff --git a/src/prompts/__init__.py b/src/prompts/__init__.py deleted file mode 100644 index 7090374..0000000 --- a/src/prompts/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Prompt templates for InsightWeaver synthesis pipeline. - -ANALYSIS_RULES.md is loaded from disk and injected into every synthesis prompt. -Frame discovery and synthesis prompts are defined as Python constants for structured output. -""" - -from pathlib import Path - - -def load_analysis_rules() -> str: - """Load ANALYSIS_RULES.md from the prompts directory.""" - rules_path = Path(__file__).parent / "ANALYSIS_RULES.md" - return rules_path.read_text() diff --git a/src/prompts/decisions.py b/src/prompts/decisions.py deleted file mode 100644 index db5d310..0000000 --- a/src/prompts/decisions.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -Prompt for the decision router. - -DECISION_ROUTING_PROMPT matches today's situations against the user's open -decision factors: which factors does each situation bear on, and which way -did the evidence move them? -""" - -DECISION_ROUTING_PROMPT = """You are routing today's news analysis into a user's standing decisions. The user is carrying a set of decisions, each with factors they are tracking. Your job is to find where today's situations contain evidence that bears on those factors. - -## How to route - -For each situation, ask: does it contain evidence that bears on any of the tracked factors? Most situations bear on no factors -- that is fine and expected. Only route a situation to a factor when there is a real, specific connection. - -When you do route: - -- **direction** -- how the evidence moves the factor: - - `supports`: the evidence makes the factor look more favorable for the decision, or confirms the user's tracked read. - - `complicates`: the evidence makes the factor look less favorable, or cuts against the user's tracked read. - - `neutral`: the evidence is relevant but does not clearly push either way. -- **epistemic_status** -- the strength of the evidence, copied from how the situation labels it: `reported_fact`, `single_source`, `consensus`, or `speculation`. When the situation is ambiguous, choose the weaker label. -- **excerpt** -- one sentence, drawn from the situation, stating what the evidence actually is. Concrete, not a paraphrase of the factor. - -Be conservative. A vague topical overlap is not evidence. If a situation discusses housing and a factor mentions housing, that is not automatically a route -- the situation must contain something that actually bears on that specific factor. - -## Open decision factors - -{factors_block} - -## Today's situations - -{situations_block} - -## Output - -Return valid JSON. Include only real routes; an empty list is a valid answer. - -```json -{{ - "evidence": [ - {{ - "situation_index": 0, - "factor_id": 5, - "direction": "complicates", - "epistemic_status": "single_source", - "excerpt": "One concrete sentence stating the evidence." - }} - ] -}} -``` - -Return ONLY the JSON, no markdown fencing or commentary.""" diff --git a/src/prompts/frames.py b/src/prompts/frames.py deleted file mode 100644 index 8e7b28f..0000000 --- a/src/prompts/frames.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -Frame discovery and frame-aware synthesis prompts. - -These prompts are used by the synthesis pipeline when interacting with topic clusters -and the narrative_frames glossary. -""" - -# Used when a topic cluster has no known frames yet. -# The model identifies candidate frames from the article corpus. -FRAME_DISCOVERY_PROMPT = """You are analyzing a cluster of articles about a single topic. No narrative frames have been catalogued for this topic yet. Your task is to identify the distinct narrative layers present in (and absent from) this coverage, and map how they relate to each other. - -A "frame" is a coherent way of understanding a story: what it emphasizes, what it de-emphasizes, and what assumption it takes for granted. Frames are not opinions -- they are structural features of how information is organized. There is never just one frame; multiple narrative layers coexist and may conflict. - -## Instructions - -Read the articles below and identify 2-4 distinct frames. For each frame: - -1. **Label**: A short, neutral name (e.g., "national security frame", "consumer welfare frame", "labor market frame"). -2. **Emphasizes**: What this frame foregrounds -- the facts, actors, or dynamics it treats as central. -3. **De-emphasizes**: What this frame pushes to the background or omits entirely. -4. **Assumed premise**: The unstated belief a reader must hold for this frame to feel natural. - -After identifying individual frames, analyze their relationships: - -5. **Fractures**: Where do the identified frames conflict? What does one frame assert that another denies or ignores? Name the specific point of disagreement. -6. **Bridges**: Is there any claim or evidence that connects conflicting frames? If no bridge exists, say so. - -Then consider what frames are plausibly absent. Absent frames may be missing because of data gaps, coverage gaps, or structural absences -- the information environment makes that perspective difficult to articulate. Name the reason. - -For absent frames, recommend a type of feed source that would carry that perspective. This must be a source category (e.g., "trade union publications", "municipal finance reporting", "patient advocacy press"), not a search query or URL. - -## Output Format - -Return valid JSON with this exact structure: - -```json -{{ - "topic_summary": "One sentence describing the topic cluster.", - "frames_present": [ - {{ - "label": "frame name", - "emphasizes": "what this frame foregrounds", - "de_emphasizes": "what this frame backgrounds", - "assumed_premise": "the unstated assumption" - }} - ], - "narrative_conflicts": [ - {{ - "frames_in_tension": ["frame label 1", "frame label 2"], - "fracture_point": "the specific disagreement between these frames", - "bridge": "what connects them, or 'none identified'" - }} - ], - "frames_absent": [ - {{ - "label": "frame name", - "emphasizes": "what this frame would foreground", - "de_emphasizes": "what this frame would background", - "assumed_premise": "the unstated assumption", - "why_absent": "data gap, coverage gap, or structural absence -- and why" - }} - ], - "gap_recommendations": [ - {{ - "absent_frame_label": "which absent frame this addresses", - "feed_type": "category of source that would carry this perspective", - "rationale": "why this source type would fill the gap" - }} - ] -}} -``` - -## Articles - -{articles}""" - - -# Maps each article in a cluster to the frame it most exhibits. -# Used to populate the article_frames table so the diet view can show -# which frames each feed carries. -FRAME_CLASSIFICATION_PROMPT = """You are tagging articles with the narrative frame each one most exhibits. A frame is a coherent way of organizing a story: what it emphasizes and what it takes for granted. - -## Frames - -{frames_block} - -## Instructions - -For each article below, decide which single frame from the list above it most exhibits, and how confident you are (0.0 to 1.0). If an article does not clearly exhibit any of the listed frames, omit it from the output -- do not force a match. - -## Articles - -{articles_block} - -## Output - -Return valid JSON: - -```json -{{ - "classifications": [ - {{ - "article_index": 0, - "frame_label": "exact label from the frames list above", - "confidence": 0.8 - }} - ] -}} -``` - -Return ONLY the JSON, no markdown fencing or commentary.""" - - -# Injected into the synthesis prompt when known frames exist for a topic. -# Asks the model to tag coverage against known frames and flag absences. -FRAME_AWARE_SYNTHESIS_PROMPT = """## Known Narrative Frames for "{topic_name}" - -The following frames have been previously identified and validated for this topic. Use them to structure your analysis. - -{frames_yaml} - -### Instructions - -For this topic in your synthesis: - -1. **Tag each article's framing** against the known frames above. An article may use more than one frame. -2. **Report frame distribution**: Which frames are represented in today's coverage and roughly how many articles use each. -3. **Flag absent frames**: Which of the known frames have zero coverage today. List them explicitly. Do not speculate about what the absent perspective would say -- the absence itself is the signal. -4. **Note novel frames**: If an article uses a frame not listed above, describe it using the same structure (label, emphasizes, de-emphasizes, assumed premise). This is a candidate for the frame glossary. - -Include the frame analysis in your output under a "frame_analysis" key for this topic.""" diff --git a/src/prompts/meta_frames.py b/src/prompts/meta_frames.py deleted file mode 100644 index 356f5ac..0000000 --- a/src/prompts/meta_frames.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Prompt for cross-cluster reconciliation. - -META_FRAME_RECONCILIATION_PROMPT looks across all of today's situations and -flags meta-fractures: a single underlying frame conflict appearing across -multiple, topically distinct situations. -""" - -META_FRAME_RECONCILIATION_PROMPT = """You are looking at frame analyses from several distinct situations in today's coverage. Each situation has its own narrative layers and its own fractures. Sometimes the same underlying fracture surfaces in more than one situation -- in different topical guises but with the same structural disagreement. - -Your job is to find those cases, and only those. - -## What counts - -A meta-fracture is a single underlying disagreement that appears across two or more situations. Examples: - -- A "scarcity vs. abundance of labor" fracture might surface in immigration coverage AND in industrial-policy coverage AND in housing coverage. -- A "national interest vs. global cooperation" fracture might surface in trade coverage AND in climate coverage. - -Two situations sharing topical overlap does not make a meta-fracture. The fracture itself -- the structural point of disagreement -- must be the same. - -Most days will have few meta-fractures or none. An empty result is a valid and expected answer. - -## Today's situations - -{situations_block} - -## Output - -Return valid JSON. Include only real meta-fractures; an empty list is fine. - -```json -{{ - "meta_fractures": [ - {{ - "name": "Short label for the underlying fracture (under 8 words).", - "description": "One sentence stating the structural disagreement.", - "situation_indices": [0, 2, 5], - "shared_point": "The specific point of disagreement that recurs across these situations." - }} - ] -}} -``` - -Return ONLY the JSON, no markdown fencing or commentary.""" diff --git a/src/prompts/predictions.py b/src/prompts/predictions.py deleted file mode 100644 index bbcb85c..0000000 --- a/src/prompts/predictions.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -Prompt for the prediction tracker. - -PREDICTION_CHECK_PROMPT grades open predictions against fresh coverage: -did the observable appear, did coverage explicitly contradict it, or is it -still open? -""" - -PREDICTION_CHECK_PROMPT = """You are checking a ledger of open predictions against today's news coverage. Each prediction is a falsifiable observable that a past analysis flagged as worth watching. Your job is to decide, for each one, whether today's coverage resolves it. - -## Verdicts - -For each prediction, return exactly one verdict: - -- **triggered**: today's coverage reports that the observable happened, or that the trigger condition was met. There must be specific evidence in the coverage -- not an inference. -- **contradicted**: today's coverage reports something that explicitly rules out the observable or the trigger condition. Again, specific evidence, not an inference. -- **open**: today's coverage does not bear on this prediction either way. This is the default. Most predictions on most days are "open". - -Be conservative. "open" is the safe answer. Only return "triggered" or "contradicted" when the coverage directly addresses the observable. A prediction about a Fed rate decision is not triggered by an article that merely discusses inflation. - -## Open predictions - -{predictions_block} - -## Today's coverage - -{coverage_block} - -## Output - -Return valid JSON, one entry per prediction: - -```json -{{ - "verdicts": [ - {{ - "prediction_id": 12, - "verdict": "triggered", - "note": "Short evidence-grounded reason, under 25 words. Name the coverage that resolves it." - }}, - {{ - "prediction_id": 13, - "verdict": "open", - "note": "No coverage bears on this." - }} - ] -}} -``` - -Return ONLY the JSON, no markdown fencing or commentary.""" diff --git a/src/prompts/questions.py b/src/prompts/questions.py deleted file mode 100644 index 4b95be6..0000000 --- a/src/prompts/questions.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -Prompts for the question matcher. - -QUESTION_MATCHING_PROMPT decides whether a freshly-proposed unresolved_question -from today's synthesis is the same underlying question as one of the currently -open Questions in the persistent graph. -""" - -QUESTION_MATCHING_PROMPT = """You are matching freshly-proposed questions against a list of open questions that the system is already tracking. Your job is to decide, for each proposed question, whether it is the same underlying question as one of the open ones, or whether it is a genuinely new question. - -## What counts as "the same question" - -Two questions are the same when they ask about the same underlying outcome, even if the phrasing differs and even if the proposed question is more specific or more general. Examples: - -- "Will the Fed cut rates in June?" and "Does the Fed pivot at the June meeting?" — same. -- "Will the bill pass?" and "Does the Senate vote it through?" — different. The second is one specific path to the first; the first encompasses the second plus other paths. -- "Will inflation come down?" and "Will the May CPI print show inflation easing?" — different. The second is a specific observation that bears on the first. - -When in doubt, prefer "new" over "match". A false match collapses two distinct epistemic threads; a false new just creates a duplicate that can be merged later. - -## Constraints - -- Never match across substantively different domains (a question about labor markets is not the same as one about housing markets, even if both touch employment). -- A proposed question that asks about a specific event (a vote, a print, a hearing) is not the same as an open question about the broader outcome that event would inform. -- The open list is sorted with most-recent first. Older open questions are likelier to be stale or already resolved-in-effect; be slightly more cautious about matching them. - -## Input - -### Proposed questions -{proposed_block} - -### Open questions -{open_block} - -## Output - -Return valid JSON with this exact structure, one entry per proposed question, in the same order: - -```json -{{ - "matches": [ - {{ - "proposed_index": 0, - "matched_id": 47, - "reasoning": "Short reason (under 20 words)." - }}, - {{ - "proposed_index": 1, - "matched_id": null, - "reasoning": "Short reason (under 20 words)." - }} - ] -}} -``` - -`matched_id` is the integer id of the matched open question, or `null` if none matches. Return ONLY the JSON, no markdown fencing or commentary.""" diff --git a/src/prompts/synthesis.py b/src/prompts/synthesis.py deleted file mode 100644 index f1ecefe..0000000 --- a/src/prompts/synthesis.py +++ /dev/null @@ -1,160 +0,0 @@ -""" -Clustering and situation synthesis prompts. - -Pass 1: CLUSTERING_PROMPT groups articles into topic clusters. -Pass 2: SITUATION_SYNTHESIS_PROMPT produces examined narratives for each cluster. -THIN_COVERAGE_PROMPT produces one-line summaries for clusters with insufficient coverage. -""" - -# Pass 1: Group articles into topic clusters. -# Input: article titles + first paragraphs. -# Output: clusters with article IDs and one-sentence descriptions. -# No significance judgment -- just topical grouping. -CLUSTERING_PROMPT = """You are grouping news articles into topic clusters. A cluster is a set of articles about related topics where understanding one informs understanding the others. Articles do not need to be about the exact same event -- they belong together if they are part of the same broader story, involve the same actors, or affect the same systems. - -## Instructions - -Read the article summaries below. Group them into **5-15 clusters**. Each article belongs to exactly one cluster. Prefer fewer, larger clusters over many single-article clusters. Two articles about different aspects of the same policy area, the same institution, or the same conflict belong in the same cluster. - -Only create a single-article cluster when an article is genuinely unrelated to everything else. - -Do not judge significance. Do not rank or prioritize. Just group by relatedness. - -## Output Format - -Return valid JSON with this exact structure: - -```json -{{ - "clusters": [ - {{ - "title": "One-sentence description of the situation", - "article_ids": [1, 3, 7], - "keywords": ["keyword1", "keyword2", "keyword3"] - }} - ] -}} -``` - -## Articles - -{articles}""" - - -# Pass 2: Situation synthesis for clusters with 3+ articles. -# Produces examined narratives: actors, interests, power dynamics, -# coverage frame, causal structure, information gaps. -# Epistemic labels on every claim. No confidence scores. -SITUATION_SYNTHESIS_PROMPT = """You are analyzing a set of articles about a single situation. The reader should finish your analysis understanding five things: - -1. What is actually happening (not just what is being reported) -2. Who is doing what to whom, and why -3. What the coverage makes easy to see and what it makes hard to see -4. Where the story could go, and what would have to be true for each path -5. What they would need to know to make a good decision about this - -{analysis_rules} - -## Citation Discipline - -For every factual claim, include inline citations using this format: "claim^[1,3]" referencing article numbers from the list below. Only cite articles that directly support the claim. Do not cite your own analytical conclusions. - -## Article Reference List -{article_ref_list} - -## Output Format - -Return valid JSON with this exact structure: - -```json -{{ - "title": "One-sentence description of the situation", - "narrative": "2-4 paragraph examined narrative. Identify the operative narrative layers -- the distinct stories being told about what is happening. Map where they conflict: what does one narrative assert that another denies or ignores? Name actors and their interests. State who benefits, who is harmed, who decides. When actors operate under conflicting narratives, name the discrepancy. Every factual claim carries an epistemic label and citation. End with the unresolved question that determines where this goes.", - "actors": [ - {{ - "name": "Named person or organization", - "role": "What they are doing in this situation", - "interests": "What they stand to gain or lose", - "epistemic_status": "reported_fact | single_source | consensus | speculation" - }} - ], - "power_dynamics": {{ - "who_benefits": "Named, with explanation^[citations]", - "who_is_harmed": "Named, with explanation^[citations]", - "who_decides": "Named, with explanation^[citations]" - }}, - "coverage_frame": {{ - "narrative_layers": "The distinct narratives operative in the coverage -- not just one dominant frame but the layers that coexist and their relationship to each other", - "fractures": "Where these narratives conflict -- the specific point of disagreement, not just the fact that disagreement exists", - "bridges": "What connects conflicting narratives, if anything. If no bridge exists, say so.", - "structural_absences": "What is hard to see -- perspectives that are absent not because of missing data but because the information environment makes them difficult to articulate. Name the structural reason." - }}, - "where_this_goes": {{ - "branching_paths": "The conditional futures: if X then Y, if A then B. Which narrative layer predicts which outcome.", - "unresolved_questions": {{ - "primary": "The single question whose answer most determines which path the situation takes. This is often the most valuable part of the analysis. Phrase as a question.", - "secondary": ["0-3 additional unresolved questions, included only if genuinely open and material. Phrase each as a question. Omit the field or pass an empty array when there are none."] - }}, - "what_to_watch": [ - {{ - "observable": "A specific, concrete event or data point that future coverage could report on -- not a vague trend. Must be falsifiable.", - "trigger_condition": "What value or outcome of that observable would signal which path is unfolding. State it so a later run can check 'did this happen'." - }} - ] - }}, - "information_gaps": [ - {{ - "what_is_missing": "Specific information or perspective not present", - "why_it_matters": "How this gap affects the reader's ability to understand the situation or make a decision", - "why_missing": "Is this a data gap, a coverage gap, or a structural absence?", - "feed_recommendation": "Type of source that would fill this gap (source category, not URL)" - }} - ], - "article_citations": [1, 3, 7] -}} -``` - -## Requirements - -- Every factual claim must carry an epistemic status label: reported fact, single-source claim, consensus view, or speculation. -- Identify multiple narrative layers, not just one dominant frame. Map their conflicts and any bridges between them. -- When the evidence is ambiguous, say so. Do not pick a side or retreat into "there are many perspectives." -- The where_this_goes section maps conditional futures, not predictions. "If X, then Y" -- not "Y will happen." -- Do not recommend actions. Map implications so the reader can reason for themselves. -- Do not use emotional language or urgency framing unless directly quoting a source. - -Return ONLY valid JSON, no markdown formatting or additional text.""" - - -# Thin coverage summary for clusters with 1-2 articles. -# Minimal output: title, source, one-line description. -THIN_COVERAGE_PROMPT = """Summarize each of these article clusters in one sentence. These clusters have insufficient coverage for full analysis (1-2 articles each). - -For each cluster, return: -- title: one-sentence description of the topic -- article_count: number of articles -- sources: list of source names -- note: one sentence on why this might matter despite thin coverage, or "Insufficient coverage for assessment" - -## Clusters - -{clusters} - -## Output Format - -Return valid JSON: - -```json -{{ - "thin_coverage": [ - {{ - "title": "Topic description", - "article_count": 1, - "sources": ["Source Name"], - "note": "One sentence." - }} - ] -}} -``` - -Return ONLY valid JSON.""" diff --git a/src/render/__init__.py b/src/render/__init__.py deleted file mode 100644 index 305fb47..0000000 --- a/src/render/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Brief rendering. - -One document model (:class:`BriefDocument`) and one renderer per output -medium. Nothing in this package fetches, synthesizes, or calls Claude -- a -``BriefDocument`` is either handed over by the pipeline or loaded from a -stored ``narrative_syntheses`` row, and rendering it is pure. -""" - -from .document import BriefDocument, StoredBriefNotFound, load_stored_brief -from .email import EmailDeliveryError, EmailRenderer -from .html import HTMLRenderer -from .markdown import MarkdownRenderer -from .terminal import TerminalRenderer, clean_citations - -__all__ = [ - "BriefDocument", - "EmailDeliveryError", - "EmailRenderer", - "HTMLRenderer", - "MarkdownRenderer", - "StoredBriefNotFound", - "TerminalRenderer", - "clean_citations", - "load_stored_brief", -] diff --git a/src/render/_text.py b/src/render/_text.py deleted file mode 100644 index ce799c6..0000000 --- a/src/render/_text.py +++ /dev/null @@ -1,300 +0,0 @@ -""" -Shared, pure text extraction for the brief renderers. - -Every renderer reads the same fields out of a synthesis payload the same way. -These helpers are that reading, defined once. They contain no styling: they -return plain strings, and each renderer decides how to present them. -""" - -from __future__ import annotations - -import re -from typing import Any - -# Glyphs for how a factor moved: shared by every renderer that shows decisions. -DIRECTION_GLYPH = { - "supports": "+", - "complicates": "-", - "neutral": "~", -} - - -def clean_citations(text: str) -> str: - """Convert ^[N,M] citation markers to [N,M] for display.""" - if not text: - return text - return re.sub(r"\^\[([0-9,\s]+)\]", r"[\1]", text) - - -def decision_summary(metadata: dict) -> list[dict]: - """Pull the decision-routing summary out of synthesis metadata.""" - routing = metadata.get("decision_routing") - return routing if isinstance(routing, list) else [] - - -def prediction_check_line(check: dict | None) -> str: - """One-line transparency summary of the pre-synthesis prediction check.""" - if not isinstance(check, dict): - return "" - checked = check.get("checked", 0) - if not checked and not check.get("expired"): - return "" - triggered = check.get("triggered", 0) - contradicted = check.get("contradicted", 0) - still_open = check.get("still_open", 0) - expired = check.get("expired", 0) - return ( - f"Prediction check: {checked} open observables graded -- " - f"{triggered} triggered, {contradicted} contradicted, " - f"{still_open} still open, {expired} expired. " - f"See 'predictions track-record'." - ) - - -# --- institutional activity ------------------------------------------------- -# -# The section reports movement against a trailing average, never a tally. A -# flat count is noise: the entities a beat declares are the ones that appear -# most days, so "FedRAMP PMO: 6" reproduces a standing fact every morning. The -# sentences below are built to read like an analyst noting a change, and the -# ordering deliberately never sorts by count -- that would be a leaderboard, -# and activity is not significance. - -MOVEMENT_UP = "up" -MOVEMENT_DOWN = "down" -MOVEMENT_UNCHANGED = "unchanged" -MOVEMENT_FIRST_RUN = "first_run" - -ACTIVITY_NOTE = ( - "Movement against each entity's trailing average. " - "A count is an observation, not a measure of significance." -) - - -def institutional_activity(metadata: dict) -> dict[str, Any]: - """ - Pull the institutional activity block out of synthesis metadata. - - Returns ``{}`` when the run recorded none -- the default person brief, or a - beat with no ``coverage`` entities. - """ - block = metadata.get("institutional_activity") - return block if isinstance(block, dict) else {} - - -def _activity_entries(block: dict) -> list[dict]: - entries = block.get("entities") - if not isinstance(entries, list): - return [] - return [entry for entry in entries if isinstance(entry, dict)] - - -def split_activity(block: dict) -> tuple[list[dict], list[dict]]: - """ - Partition activity entries into ``(moved, steady)``, order preserved. - - ``moved`` carries the entities whose count departed from their baseline, - plus the ones being observed for the first time (which have no baseline to - depart from and are stated as such). ``steady`` carries the rest: entities - that have been active before and are behaving normally, including normally - quiet ones. They are kept because an entity going silent is information, - and dropping it is the same class of bug as a standing question vanishing - on a quiet day. - """ - moved: list[dict] = [] - steady: list[dict] = [] - for entry in _activity_entries(block): - movement = entry.get("movement") - if movement in (MOVEMENT_UP, MOVEMENT_DOWN, MOVEMENT_FIRST_RUN): - moved.append(entry) - else: - steady.append(entry) - return moved, steady - - -def _format_average(value: Any) -> str: - """Render a trailing average without trailing zeros: 1.0 -> "1".""" - if not isinstance(value, int | float) or isinstance(value, bool): - return "0" - return f"{round(float(value), 1):g}" - - -def _items(count: int) -> str: - return "1 item" if count == 1 else f"{count} items" - - -def activity_sentence(entry: dict) -> str: - """ - One entity's reading as a sentence. - - Three forms, because there are three things that can be true: it moved - against a baseline, it has no baseline yet, or it is where it usually is. - """ - name = str(entry.get("name", "(entity)")) - raw_count = entry.get("count", 0) - count = raw_count if isinstance(raw_count, int) and not isinstance(raw_count, bool) else 0 - movement = entry.get("movement") - - if movement == MOVEMENT_FIRST_RUN: - return f"{name} appeared in {_items(count)} this run; no trailing average yet." - if movement in (MOVEMENT_UP, MOVEMENT_DOWN): - average = _format_average(entry.get("trailing_average")) - return ( - f"{name} appeared in {_items(count)} this run, against a trailing average of {average}." - ) - return f"{name} appeared in {count}, unchanged." - - -def activity_footnote(block: dict) -> str: - """ - One line accounting for the entities the section does not list. - - An entity with no mentions and no history does not appear -- declaring it - was a hypothesis about where news comes from, and a hypothesis that has - never paid out is a note about the config rather than a line in the brief. - Saying how many were withheld keeps that omission visible without naming - them into the brief. - """ - withheld = block.get("never_observed") - if not isinstance(withheld, int) or isinstance(withheld, bool) or withheld <= 0: - return "" - if withheld == 1: - return "1 declared entity has never been mentioned and is not listed." - return f"{withheld} declared entities have never been mentioned and are not listed." - - -def standing_agenda(metadata: dict) -> list[dict]: - """ - Pull this beat's standing-agenda review out of synthesis metadata. - - Returns every entry the run recorded, including the ones that did not move. - Renderers must not filter this list: an unmoved standing question is the - finding, and dropping it is the failure the standing agenda exists to - prevent (added 2026-08-26, backlog task 007). - """ - agenda = metadata.get("standing_agenda") - if not isinstance(agenda, list): - return [] - return [entry for entry in agenda if isinstance(entry, dict)] - - -def standing_agenda_status(entry: dict) -> str: - """The short label for how one standing question moved: MOVED or NO MOVEMENT.""" - return "MOVED" if entry.get("moved") else "NO MOVEMENT" - - -def standing_agenda_provenance(entry: dict) -> str: - """ - The identity line under a standing question: which Q it is, how many runs - of this beat have moved it, and when it last did. - """ - parts = [f"Q{entry.get('question_id', '?')}"] - declared = entry.get("declared_at") - if declared: - parts.append(f"declared {declared}") - - count = entry.get("appearance_count") or 0 - if entry.get("moved"): - parts.append(f"run {count}") - elif count: - last = entry.get("last_moved_at") - parts.append( - f"{count} prior run(s), last moved {last}" if last else f"{count} prior run(s)" - ) - else: - parts.append("never moved") - - status = entry.get("status") - if status and status != "open": - parts.append(str(status)) - return f"{parts[0]} ({', '.join(parts[1:])})" if len(parts) > 1 else parts[0] - - -def standing_agenda_movement(entry: dict) -> list[str]: - """ - What moved this standing question in this run, one line per situation. - - Empty when nothing moved -- the caller says so explicitly rather than - rendering nothing. - """ - lines: list[str] = [] - for item in entry.get("moved_in") or []: - if not isinstance(item, dict): - continue - index = item.get("situation_index") - title = clean_citations(str(item.get("title") or "")).strip() - label = f"Situation {index}" if index is not None else "This run" - lines.append(f"{label}: {title}" if title else label) - return lines - - -def standing_agenda_no_movement(entry: dict) -> str: - """The sentence shown when a standing question did not move this run.""" - last = entry.get("last_moved_at") - if last: - return f"No coverage this run bore on this question. Last moved {last}." - return "No coverage this run bore on this question, and none ever has." - - -def watch_items(futures: dict) -> list[str]: - """Extract what_to_watch observables. Handles the list-of-objects shape - and the legacy single-string shape.""" - watch = futures.get("what_to_watch") - if isinstance(watch, str): - return [watch] if watch.strip() else [] - if not isinstance(watch, list): - return [] - items: list[str] = [] - for entry in watch: - if isinstance(entry, str) and entry.strip(): - items.append(entry.strip()) - elif isinstance(entry, dict): - observable = (entry.get("observable") or "").strip() - trigger = (entry.get("trigger_condition") or "").strip() - if observable and trigger: - items.append(f"{observable} -- {trigger}") - elif observable: - items.append(observable) - return items - - -def question_lines( - futures: dict, -) -> tuple[str, str, list[tuple[str, str]]]: - """ - Extract primary question text + identity prefix and secondary lines from - a ``where_this_goes`` block. Returns ``(primary_text, primary_prefix, - secondary)`` where each secondary is ``(text, prefix)``. Prefix is empty - when a question has no identity metadata or is appearing for the first - time -- accumulation context is only shown for repeat appearances. - """ - uq = futures.get("unresolved_questions") - if not isinstance(uq, dict): - legacy = futures.get("unresolved_question") - if isinstance(legacy, str): - return legacy, "", [] - return "", "", [] - - def _split(entry: Any) -> tuple[str, str]: - if isinstance(entry, str): - return entry, "" - if not isinstance(entry, dict): - return "", "" - text = entry.get("text", "") - qid = entry.get("question_id") - appearance = entry.get("appearance_count") - if qid is None or appearance is None: - return text, "" - if appearance <= 1: - return text, f"Q{qid} (new)" - first = entry.get("first_asked_at", "") - first_date = first.split("T", 1)[0] if isinstance(first, str) else "" - return text, f"Q{qid} (run {appearance}, asked {first_date})" - - primary_text, primary_prefix = _split(uq.get("primary")) - secondary: list[tuple[str, str]] = [] - for entry in uq.get("secondary") or []: - text, prefix = _split(entry) - if text: - secondary.append((text, prefix)) - return primary_text, primary_prefix, secondary diff --git a/src/render/document.py b/src/render/document.py deleted file mode 100644 index 7405af6..0000000 --- a/src/render/document.py +++ /dev/null @@ -1,281 +0,0 @@ -""" -BriefDocument -- the single structured model every brief renderer consumes. - -The pipeline already produces exactly this content as ``synthesis_data`` and -already stores it in ``narrative_syntheses.synthesis_data``. This module does -not reshape or reinterpret that content; it names it, so that a renderer can -take a document rather than an untyped nest of dicts, and so that a stored run -can be replayed without touching the pipeline. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any - - -class StoredBriefNotFound(LookupError): - """Raised when ``--from-run`` names a synthesis id that is not stored.""" - - def __init__(self, synthesis_id: int, available: list[int]) -> None: - self.synthesis_id = synthesis_id - self.available = available - detail = ", ".join(str(i) for i in available) if available else "none" - super().__init__( - f"No stored brief with id {synthesis_id}. Most recent stored ids: {detail}" - ) - - -@dataclass(frozen=True) -class BriefDocument: - """ - A rendered-brief-shaped view of one synthesis. - - ``situations``, ``thin_coverage``, ``meta_fractures`` and ``metadata`` are - the synthesis payload verbatim. The remaining fields are provenance: which - stored row this came from, and how many articles the run analyzed. - """ - - situations: list[dict[str, Any]] = field(default_factory=list) - thin_coverage: list[dict[str, Any]] = field(default_factory=list) - meta_fractures: list[dict[str, Any]] = field(default_factory=list) - metadata: dict[str, Any] = field(default_factory=dict) - articles_analyzed: int = 0 - synthesis_id: int | None = None - analysis_run_id: int | None = None - generated_at: str | None = None - - # -- construction ---------------------------------------------------- - - @classmethod - def from_synthesis_data( - cls, - synthesis_data: dict[str, Any] | None, - *, - articles_analyzed: int | None = None, - synthesis_id: int | None = None, - analysis_run_id: int | None = None, - generated_at: str | None = None, - ) -> BriefDocument: - """Build a document from a raw ``synthesis_data`` payload.""" - data = synthesis_data or {} - metadata = _as_dict(data.get("metadata")) - if articles_analyzed is None: - articles_analyzed = _as_int(metadata.get("articles_analyzed")) - if generated_at is None: - raw = metadata.get("generated_at") - generated_at = raw if isinstance(raw, str) else None - return cls( - situations=_as_dict_list(data.get("situations")), - thin_coverage=_as_dict_list(data.get("thin_coverage")), - meta_fractures=_as_dict_list(data.get("meta_fractures")), - metadata=metadata, - articles_analyzed=articles_analyzed, - synthesis_id=synthesis_id, - analysis_run_id=analysis_run_id, - generated_at=generated_at, - ) - - @classmethod - def from_report(cls, report_data: dict[str, Any] | None) -> BriefDocument: - """ - Build a document from the legacy pipeline ``report_result`` shape: - ``{"synthesis_data": ..., "articles_analyzed": ..., "synthesis_id": ...}``. - """ - report = report_data or {} - raw_articles = report.get("articles_analyzed") - return cls.from_synthesis_data( - _as_dict(report.get("synthesis_data")), - articles_analyzed=_as_int(raw_articles) if raw_articles is not None else None, - synthesis_id=_as_optional_int(report.get("synthesis_id")), - ) - - def to_report(self) -> dict[str, Any]: - """Round-trip back to the legacy ``report_result`` shape.""" - return { - "success": True, - "articles_analyzed": self.articles_analyzed, - "synthesis_data": self.synthesis_data, - "synthesis_id": self.synthesis_id, - } - - # -- payload --------------------------------------------------------- - - @property - def synthesis_data(self) -> dict[str, Any]: - """The synthesis payload, in the shape the pipeline stores.""" - return { - "situations": self.situations, - "thin_coverage": self.thin_coverage, - "meta_fractures": self.meta_fractures, - "metadata": self.metadata, - } - - def is_empty(self) -> bool: - """True when no section carries anything worth rendering.""" - return not (self.situations or self.thin_coverage or self.meta_fractures) - - # -- named views over metadata -------------------------------------- - - @property - def decisions(self) -> list[dict[str, Any]]: - """Decision-routing summary: what today's coverage moved.""" - return _as_dict_list(self.metadata.get("decision_routing")) - - @property - def standing_agenda(self) -> list[dict[str, Any]]: - """ - This beat's declared agenda and how it moved, as the run recorded it. - - Empty for the person brief and for any beat that declares nothing. Note - that a non-empty list can contain entries that did not move; that is - the point, and no consumer may drop them. - """ - return _as_dict_list(self.metadata.get("standing_agenda")) - - @property - def prediction_check(self) -> dict[str, Any]: - """Pre-synthesis grading of open observables.""" - return _as_dict(self.metadata.get("prediction_check")) - - @property - def institutional_activity(self) -> dict[str, Any]: - """ - This run's institutional coverage reading: which of the beat's declared - orgs, programs and document types moved against their trailing average. - - Empty for the default person brief and for a beat that declares no - ``coverage`` entities. - """ - return _as_dict(self.metadata.get("institutional_activity")) - - @property - def clusters_analyzed(self) -> int: - return _as_int(self.metadata.get("clusters_analyzed")) - - @property - def clusters_thin(self) -> int: - return _as_int(self.metadata.get("clusters_thin")) - - @property - def clusters_total(self) -> int: - return _as_int(self.metadata.get("clusters_total")) - - @property - def analysis_threshold(self) -> str: - threshold = self.metadata.get("analysis_threshold") - return threshold if isinstance(threshold, str) else "3+ articles" - - @property - def date_stamp(self) -> str: - """ - Date this brief was generated, ``YYYY-MM-DD``, or empty when unknown. - - Taken from stored provenance, never from the clock -- rendering the same - stored run tomorrow must produce the same bytes. - """ - raw = self.generated_at or self.metadata.get("generated_at") - if not isinstance(raw, str) or not raw.strip(): - return "" - return raw.split("T", 1)[0].split(" ", 1)[0] - - @property - def metadata_articles_analyzed(self) -> int: - """Article count as the synthesis itself recorded it.""" - return _as_int(self.metadata.get("articles_analyzed")) - - @property - def questions(self) -> list[dict[str, Any]]: - """ - Every unresolved question raised across situations, in situation order, - primary before secondary. Each entry carries ``situation_index`` (1-based) - and ``role`` ("primary" or "secondary") alongside the question payload. - """ - collected: list[dict[str, Any]] = [] - for index, situation in enumerate(self.situations, 1): - futures = _as_dict(situation.get("where_this_goes")) - unresolved = futures.get("unresolved_questions") - if not isinstance(unresolved, dict): - legacy = futures.get("unresolved_question") - if isinstance(legacy, str) and legacy.strip(): - collected.append({"situation_index": index, "role": "primary", "text": legacy}) - continue - primary = _question_entry(unresolved.get("primary")) - if primary: - collected.append({"situation_index": index, "role": "primary", **primary}) - for raw in unresolved.get("secondary") or []: - secondary = _question_entry(raw) - if secondary: - collected.append({"situation_index": index, "role": "secondary", **secondary}) - return collected - - -def _question_entry(raw: Any) -> dict[str, Any] | None: - """Normalize one question to a dict, or None when it carries no text.""" - if isinstance(raw, str): - return {"text": raw} if raw.strip() else None - if isinstance(raw, dict): - text = raw.get("text") - return dict(raw) if isinstance(text, str) and text.strip() else None - return None - - -def _as_dict(value: Any) -> dict[str, Any]: - return value if isinstance(value, dict) else {} - - -def _as_dict_list(value: Any) -> list[dict[str, Any]]: - if not isinstance(value, list): - return [] - return [item for item in value if isinstance(item, dict)] - - -def _as_int(value: Any) -> int: - return value if isinstance(value, int) and not isinstance(value, bool) else 0 - - -def _as_optional_int(value: Any) -> int | None: - return value if isinstance(value, int) and not isinstance(value, bool) else None - - -def load_stored_brief(synthesis_id: int) -> BriefDocument: - """ - Load a stored brief by ``narrative_syntheses.id``. - - Read-only and offline: no pipeline stage runs, no feed is fetched and no - Claude call is made. Raises :class:`StoredBriefNotFound` if the id is not - stored, listing recent ids so the caller can correct it. - """ - from sqlalchemy.orm import Session - - from ..database.connection import get_db - from ..database.models import NarrativeSynthesis - - db: Session - with get_db() as db: - # Annotated because src.database.models is excluded from type checking, - # so the ORM class arrives here as Any. - row: Any = ( - db.query(NarrativeSynthesis).filter(NarrativeSynthesis.id == synthesis_id).first() - ) - if row is None: - # Bound to list[Any] rather than unpacked in the comprehension: the - # column arrives untyped for the same reason `row` does, so mypy - # infers the rows of a single-column query as Never and rejects the - # tuple unpack. Indexing an Any row keeps the runtime identical. - recent: list[Any] = ( - db.query(NarrativeSynthesis.id) - .order_by(NarrativeSynthesis.id.desc()) - .limit(10) - .all() - ) - raise StoredBriefNotFound(synthesis_id, [int(entry[0]) for entry in recent]) - - generated_at = row.generated_at.isoformat() if row.generated_at else None - return BriefDocument.from_synthesis_data( - _as_dict(row.synthesis_data), - articles_analyzed=_as_int(row.articles_analyzed), - synthesis_id=row.id, - analysis_run_id=row.analysis_run_id, - generated_at=generated_at, - ) diff --git a/src/render/email.py b/src/render/email.py deleted file mode 100644 index 0f4c778..0000000 --- a/src/render/email.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -Email renderer. - -Wraps :class:`~src.render.html.HTMLRenderer`: the HTML body is the same -self-contained document the ``html`` format writes, with a plain-text -alternative derived from the terminal rendering. Delivery uses the SMTP -environment variables the project already documents in ``.env``: - - SMTP_SERVER, SMTP_PORT, EMAIL_USERNAME, EMAIL_PASSWORD, - FROM_EMAIL, RECIPIENT_EMAIL - -There is no retry and no outbox. A failed send raises -:class:`EmailDeliveryError` and the caller exits non-zero. -""" - -from __future__ import annotations - -import os -import re -import smtplib -from dataclasses import dataclass -from email.message import EmailMessage - -from .document import BriefDocument -from .html import HTMLRenderer -from .terminal import TerminalRenderer - -__all__ = ["EmailDeliveryError", "EmailRenderer", "SMTPConfig"] - -_ANSI = re.compile(r"\x1b\[[0-9;]*m") - -# Credentials are read from these but never echoed; only names appear in errors. -_REQUIRED_VARS = ("SMTP_SERVER", "EMAIL_USERNAME", "EMAIL_PASSWORD", "FROM_EMAIL") - - -class EmailDeliveryError(RuntimeError): - """Configuration or transport failure while sending a brief.""" - - -@dataclass(frozen=True) -class SMTPConfig: - """SMTP settings, read from the environment at send time.""" - - server: str - port: int - username: str - password: str - from_email: str - recipient: str - - @classmethod - def from_env(cls, recipient: str | None = None) -> SMTPConfig: - """ - Read SMTP settings from the environment. - - Raises :class:`EmailDeliveryError` naming any missing variable. Values - are never included in the message -- only variable names. - """ - missing = [name for name in _REQUIRED_VARS if not os.getenv(name)] - target = recipient or os.getenv("RECIPIENT_EMAIL") or "" - if not target: - missing.append("RECIPIENT_EMAIL") - if missing: - raise EmailDeliveryError( - "Cannot send brief: missing environment variable(s): " + ", ".join(missing) - ) - - raw_port = os.getenv("SMTP_PORT", "587") - try: - port = int(raw_port) - except ValueError: - raise EmailDeliveryError(f"SMTP_PORT is not a number: {raw_port!r}") - - return cls( - server=os.environ["SMTP_SERVER"], - port=port, - username=os.environ["EMAIL_USERNAME"], - password=os.environ["EMAIL_PASSWORD"], - from_email=os.environ["FROM_EMAIL"], - recipient=target, - ) - - -class EmailRenderer: - """Render a :class:`BriefDocument` as a sendable email message.""" - - def __init__( - self, - html_renderer: HTMLRenderer | None = None, - terminal_renderer: TerminalRenderer | None = None, - ) -> None: - self.html_renderer = html_renderer or HTMLRenderer() - self.terminal_renderer = terminal_renderer or TerminalRenderer() - - def subject(self, doc: BriefDocument) -> str: - """Subject line -- stable for a given stored run.""" - stamp = doc.date_stamp - base = f"Intelligence Brief — {stamp}" if stamp else "Intelligence Brief" - count = len(doc.situations) - return f"{base} ({count} situation{'s' if count != 1 else ''})" - - def render_text(self, doc: BriefDocument) -> str: - """Plain-text alternative: the terminal brief with color escapes removed.""" - return _ANSI.sub("", self.terminal_renderer.render(doc)) - - def render( - self, - doc: BriefDocument, - *, - from_email: str = "", - recipient: str = "", - ) -> EmailMessage: - """Build the multipart/alternative message for this brief.""" - message = EmailMessage() - message["Subject"] = self.subject(doc) - if from_email: - message["From"] = from_email - if recipient: - message["To"] = recipient - message.set_content(self.render_text(doc)) - message.add_alternative(self.html_renderer.render(doc), subtype="html") - return message - - def send( - self, - doc: BriefDocument, - *, - recipient: str | None = None, - config: SMTPConfig | None = None, - ) -> str: - """ - Send the brief. Returns the recipient address on success. - - Raises :class:`EmailDeliveryError` on any configuration or transport - failure -- no retry, no queueing. - """ - cfg = config or SMTPConfig.from_env(recipient) - message = self.render(doc, from_email=cfg.from_email, recipient=cfg.recipient) - - try: - if cfg.port == 465: - with smtplib.SMTP_SSL(cfg.server, cfg.port, timeout=30) as smtp: - smtp.login(cfg.username, cfg.password) - smtp.send_message(message) - else: - with smtplib.SMTP(cfg.server, cfg.port, timeout=30) as smtp: - smtp.starttls() - smtp.login(cfg.username, cfg.password) - smtp.send_message(message) - # smtplib.SMTPException derives from OSError, so this covers both - # protocol-level rejections and connection/DNS/TLS failures. - except OSError as exc: - raise EmailDeliveryError( - f"Sending the brief to {cfg.recipient} via {cfg.server}:{cfg.port} failed: {exc}" - ) - return cfg.recipient diff --git a/src/render/html.py b/src/render/html.py deleted file mode 100644 index d588ef8..0000000 --- a/src/render/html.py +++ /dev/null @@ -1,417 +0,0 @@ -""" -HTML renderer. - -Produces one self-contained document: inline stylesheet, no scripts, no web -fonts, no images, no external requests of any kind. Opening the file offline -renders exactly what was written. Output is a pure function of the -:class:`BriefDocument`, so rendering the same stored run twice is byte-identical. -""" - -from __future__ import annotations - -from html import escape - -from ._text import ( - ACTIVITY_NOTE, - DIRECTION_GLYPH, - activity_footnote, - activity_sentence, - clean_citations, - decision_summary, - prediction_check_line, - question_lines, - split_activity, - standing_agenda, - standing_agenda_movement, - standing_agenda_no_movement, - standing_agenda_provenance, - standing_agenda_status, - watch_items, -) -from .document import BriefDocument - -__all__ = ["HTMLRenderer"] - -# Deliberately small and self-contained: system font stack (no web fonts), -# no framework, no external assets. See OUT OF SCOPE in backlog/003. -STYLESHEET = """\ -:root { color-scheme: light dark; } -body { - margin: 0 auto; - padding: 2rem 1.25rem 4rem; - max-width: 46rem; - font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; - font-size: 16px; - line-height: 1.55; - color: #1a1a1a; - background: #ffffff; -} -h1 { font-size: 1.6rem; margin: 0 0 0.25rem; letter-spacing: 0.02em; } -h2 { font-size: 1.2rem; margin: 2.5rem 0 0.25rem; } -h3 { font-size: 0.95rem; margin: 1.5rem 0 0.35rem; text-transform: uppercase; letter-spacing: 0.06em; color: #444; } -p { margin: 0 0 0.9rem; } -ul { margin: 0 0 0.9rem; padding-left: 1.2rem; } -li { margin: 0 0 0.3rem; } -.meta { color: #555; font-size: 0.85rem; margin: 0 0 0.35rem; } -.rule { border: 0; border-top: 1px solid #d8d8d8; margin: 0.75rem 0 0; } -.section-note { color: #555; font-size: 0.85rem; font-style: italic; margin: 0 0 0.9rem; } -.situation { margin: 0 0 2.5rem; } -.label { color: #444; font-weight: 600; } -.muted { color: #555; } -.gap { color: #8a5a00; font-weight: 600; } -.tag { color: #555; font-size: 0.8rem; } -.glyph { display: inline-block; min-width: 1.4rem; font-family: ui-monospace, "SFMono-Regular", Menlo, Consolas, monospace; } -.empty { color: #555; font-style: italic; } -@media (prefers-color-scheme: dark) { - body { color: #e6e6e6; background: #161616; } - h3, .meta, .section-note, .muted, .tag { color: #a8a8a8; } - .label { color: #cfcfcf; } - .rule { border-top-color: #3a3a3a; } - .gap { color: #e0a33a; } -} -""" - - -class HTMLRenderer: - """Render a :class:`BriefDocument` as one self-contained HTML page.""" - - def render(self, doc: BriefDocument) -> str: - body = self._body(doc) - return ( - "\n" - '\n' - "\n" - '\n' - '\n' - f"{escape(self.title(doc))}\n" - f"\n" - "\n" - "\n" - f"{body}\n" - "\n" - "\n" - ) - - def title(self, doc: BriefDocument) -> str: - """Document title -- stable for a given stored run.""" - stamp = doc.date_stamp - return f"Intelligence Brief — {stamp}" if stamp else "Intelligence Brief" - - # -- sections -------------------------------------------------------- - - def _body(self, doc: BriefDocument) -> str: - out: list[str] = ["

Intelligence Brief

"] - - stamp = doc.date_stamp - if stamp: - out.append(f'

Generated {escape(stamp)}

') - - metadata = doc.metadata - out.append( - '

' - f"Articles: {escape(str(metadata.get('articles_analyzed', 0)))} | " - f"Situations: {escape(str(metadata.get('clusters_analyzed', 0)))} analyzed, " - f"{escape(str(metadata.get('clusters_thin', 0)))} thin coverage | " - f"Threshold: {escape(str(metadata.get('analysis_threshold', '3+ articles')))}" - "

" - ) - - check_line = prediction_check_line(metadata.get("prediction_check")) - if check_line: - out.append(f'

{escape(check_line)}

') - - if doc.synthesis_id is not None: - out.append(f'

Stored brief #{escape(str(doc.synthesis_id))}

') - - out.append('
') - - out.extend(self._decisions(doc)) - out.extend(self._standing_agenda(doc)) - out.extend(self._situations(doc)) - out.extend(self._meta_fractures(doc)) - out.extend(self._institutional_activity(doc)) - out.extend(self._thin_coverage(doc)) - - return "\n".join(out) - - def _decisions(self, doc: BriefDocument) -> list[str]: - decisions = decision_summary(doc.metadata) - if not decisions: - return [] - out = [ - "

Your decisions

", - '

What today\'s coverage moved.

', - ] - for entry in decisions: - out.append(f"

{escape(str(entry.get('decision', '(decision)')))}

") - factors = entry.get("factors", []) - if factors: - out.append("
    ") - for factor in factors: - direction = str(factor.get("direction", "neutral")) - glyph = DIRECTION_GLYPH.get(direction, "~") - name = str(factor.get("name", "(factor)")) - out.append( - f'
  • [{escape(glyph)}]{escape(name)} ' - f'({escape(direction)})
  • ' - ) - out.append("
") - return out - - def _institutional_activity(self, doc: BriefDocument) -> list[str]: - """ - What the beat's declared institutions did this run against their - baseline. Steady entities are rendered muted rather than dropped: an - office going quiet is information. - """ - block = doc.institutional_activity - moved, steady = split_activity(block) - if not moved and not steady: - return [] - - out = [ - "

Institutional activity

", - f'

{escape(ACTIVITY_NOTE)}

', - "
    ", - ] - for entry in moved: - out.append(f"
  • {escape(activity_sentence(entry))}
  • ") - for entry in steady: - out.append(f'
  • {escape(activity_sentence(entry))}
  • ') - out.append("
") - - footnote = activity_footnote(block) - if footnote: - out.append(f'

{escape(footnote)}

') - return out - - def _standing_agenda(self, doc: BriefDocument) -> list[str]: - """ - The beat's declared agenda, every item of it. - - As in the other renderers, an unmoved standing question is rendered - with a NO MOVEMENT label rather than omitted. - """ - agenda = standing_agenda(doc.metadata) - if not agenda: - return [] - - out = [ - "

Standing agenda

", - '

What this beat declared it is watching. ' - "Unmoved items are reported, not dropped.

", - ] - for entry in agenda: - status = standing_agenda_status(entry) - text = str(entry.get("text", "(question)")) - out.append(f"

[{escape(status)}] {escape(text)}

") - out.append(f'

{escape(standing_agenda_provenance(entry))}

') - out.append("
    ") - movement = standing_agenda_movement(entry) - if movement: - for item in movement: - out.append(f'
  • Moved in: {escape(item)}
  • ') - else: - out.append(f"
  • {escape(standing_agenda_no_movement(entry))}
  • ") - for observable in entry.get("watching") or []: - out.append( - f'
  • Watching for: {escape(str(observable))}
  • ' - ) - out.append("
") - return out - - def _situations(self, doc: BriefDocument) -> list[str]: - if not doc.situations: - return ['

No situations met the analysis threshold.

'] - out: list[str] = [] - for index, situation in enumerate(doc.situations, 1): - out.extend(self._situation(situation, index)) - return out - - def _situation(self, situation: dict, index: int) -> list[str]: - title = clean_citations(str(situation.get("title", "Untitled situation"))) - out = ['
', f"

Situation {index}: {escape(title)}

"] - - narrative = situation.get("narrative", "") - if narrative: - for paragraph in clean_citations(str(narrative)).split("\n\n"): - text = paragraph.strip() - if text: - out.append(f"

{escape(text)}

") - - actors = situation.get("actors", []) - if actors: - out.append("

Actors

") - out.append("
    ") - for actor in actors: - name = escape(str(actor.get("name", "Unknown"))) - status = str(actor.get("epistemic_status", "") or "") - status_tag = f' [{escape(status)}]' if status else "" - details: list[str] = [] - if actor.get("role"): - details.append(f"Role: {escape(str(actor['role']))}") - if actor.get("interests"): - details.append(f"Interests: {escape(str(actor['interests']))}") - detail_html = "" - if details: - detail_items = "".join(f"
  • {d}
  • " for d in details) - detail_html = f"
      {detail_items}
    " - out.append(f'
  • {name}{status_tag}{detail_html}
  • ') - out.append("
") - - power = situation.get("power_dynamics", {}) - if power: - out.append("

Power dynamics

") - out.extend( - self._definition_list( - [ - ("Benefits", power.get("who_benefits"), True), - ("Harmed", power.get("who_is_harmed"), True), - ("Decides", power.get("who_decides"), True), - ] - ) - ) - - frame = situation.get("coverage_frame", {}) - if frame: - out.append("

What the coverage makes visible and invisible

") - rows: list[tuple[str, object, bool]] = [] - if frame.get("narrative_layers"): - rows.append(("Narratives", frame.get("narrative_layers"), False)) - elif frame.get("dominant_frame"): - rows.append(("Frame", frame.get("dominant_frame"), False)) - rows.append(("Fractures", frame.get("fractures"), False)) - rows.append(("Bridges", frame.get("bridges"), False)) - rows.append(("Hard to see", frame.get("structural_absences"), False)) - rows.append(("Assumes", frame.get("assumed_premise"), False)) - out.extend(self._definition_list(rows)) - - futures = situation.get("where_this_goes", {}) - if futures: - out.append("

Where this goes

") - rows = [("Paths", futures.get("branching_paths"), True)] - out.extend(self._definition_list(rows)) - - primary_text, primary_prefix, secondary = question_lines(futures) - if primary_text or secondary: - out.append("
    ") - if primary_text: - tag = ( - f'{escape(primary_prefix)} ' - if primary_prefix - else "" - ) - out.append( - f'
  • Key question: {tag}{escape(primary_text)}
  • ' - ) - for sec_text, sec_prefix in secondary: - tag = f'{escape(sec_prefix)} ' if sec_prefix else "" - out.append(f'
  • Also open: {tag}{escape(sec_text)}
  • ') - out.append("
") - - items = watch_items(futures) - if items: - out.append("
    ") - for item in items: - out.append( - f'
  • Watch for: ' - f'{escape(item)}
  • ' - ) - out.append("
") - - causal = situation.get("causal_structure", {}) - if causal: - out.append("

Causal structure

") - out.extend( - self._definition_list( - [ - ("Forces", causal.get("forces"), True), - ("Constraints", causal.get("constraints"), True), - ("Dependencies", causal.get("dependencies"), True), - ] - ) - ) - - gaps = situation.get("information_gaps", []) - if gaps: - out.append("

Information gaps

") - out.append("
    ") - for gap in gaps: - missing = escape(str(gap.get("what_is_missing", "") or "")) - details = [] - if gap.get("why_it_matters"): - details.append(f"Why it matters: {escape(str(gap['why_it_matters']))}") - if gap.get("why_missing"): - details.append(f"Why missing: {escape(str(gap['why_missing']))}") - if gap.get("feed_recommendation"): - details.append(f"Suggested source: {escape(str(gap['feed_recommendation']))}") - detail_html = "" - if details: - items_html = "".join(f'
  • {d}
  • ' for d in details) - detail_html = f"
      {items_html}
    " - out.append(f'
  • GAP: {missing}{detail_html}
  • ') - out.append("
") - - out.append("
") - return out - - def _meta_fractures(self, doc: BriefDocument) -> list[str]: - if not doc.meta_fractures: - return [] - out = [ - "

Meta-fractures

", - '

Underlying frame conflicts that surface across situations.

', - ] - for mf in doc.meta_fractures: - out.append(f"

{escape(str(mf.get('name', '(unnamed)')))}

") - if mf.get("description"): - out.append(f"

{escape(str(mf['description']))}

") - indices = mf.get("situation_indices", []) - if indices: - labels = ", ".join(f"Situation {int(i) + 1}" for i in indices) - out.append(f'

Appears in: {escape(labels)}

') - if mf.get("shared_point"): - out.append( - f'

Shared point: ' - f'{escape(str(mf["shared_point"]))}

' - ) - return out - - def _thin_coverage(self, doc: BriefDocument) -> list[str]: - if not doc.thin_coverage: - return [] - out = [ - "

Thin coverage

", - '

Topics with 1-2 articles. Listed but not fully analyzed.

', - "
    ", - ] - for item in doc.thin_coverage: - title = escape(str(item.get("title", "Unknown topic"))) - count = escape(str(item.get("article_count", 0))) - sources = escape(", ".join(str(s) for s in item.get("sources", []))) - note = item.get("note", "") - note_html = f'
    • {escape(str(note))}
    ' if note else "" - out.append( - f'
  • {title} ' - f'{count} article(s) | {sources}{note_html}
  • ' - ) - out.append("
") - return out - - @staticmethod - def _definition_list(rows: list[tuple[str, object, bool]]) -> list[str]: - """ - Render ``(label, value, strip_citations)`` rows as a list, skipping - empty values. Returns an empty list when nothing survives. - """ - rendered = [] - for label, value, strip in rows: - if not value: - continue - text = str(value) - if strip: - text = clean_citations(text) - rendered.append(f'
  • {escape(label)}: {escape(text)}
  • ') - if not rendered: - return [] - return ["
      ", *rendered, "
    "] diff --git a/src/render/markdown.py b/src/render/markdown.py deleted file mode 100644 index ececa8e..0000000 --- a/src/render/markdown.py +++ /dev/null @@ -1,280 +0,0 @@ -""" -Markdown renderer -- the archival format behind ``brief --save PATH``. - -A move of the former ``BriefFormatter.format_markdown``, unchanged in output. -""" - -from __future__ import annotations - -from ._text import ( - ACTIVITY_NOTE, - activity_footnote, - activity_sentence, - clean_citations, - decision_summary, - institutional_activity, - prediction_check_line, - question_lines, - split_activity, - standing_agenda, - standing_agenda_movement, - standing_agenda_no_movement, - standing_agenda_provenance, - standing_agenda_status, - watch_items, -) -from .document import BriefDocument - -__all__ = ["MarkdownRenderer"] - - -class MarkdownRenderer: - """Render a :class:`BriefDocument` as markdown for archival.""" - - def render(self, doc: BriefDocument) -> str: - situations = doc.situations - thin_coverage = doc.thin_coverage - meta_fractures = doc.meta_fractures - metadata = doc.metadata - - lines: list[str] = ["# Intelligence Brief", ""] - - articles = metadata.get("articles_analyzed", 0) - clusters_analyzed = metadata.get("clusters_analyzed", 0) - clusters_thin = metadata.get("clusters_thin", 0) - threshold = metadata.get("analysis_threshold", "3+ articles") - lines.append( - f"_Articles: {articles} | Situations: {clusters_analyzed} analyzed, " - f"{clusters_thin} thin coverage | Threshold: {threshold}_" - ) - check_line = prediction_check_line(metadata.get("prediction_check")) - if check_line: - lines.append("") - lines.append(f"_{check_line}_") - lines.append("") - - decisions = decision_summary(metadata) - if decisions: - lines.append("## Your decisions") - lines.append("") - lines.append("_What today's coverage moved._") - lines.append("") - for entry in decisions: - lines.append(f"### {entry.get('decision', '(decision)')}") - for factor in entry.get("factors", []): - direction = factor.get("direction", "neutral") - name = factor.get("name", "(factor)") - lines.append(f"- **{name}** — {direction}") - lines.append("") - - lines.extend(self._render_standing_agenda(doc)) - - if situations: - for i, situation in enumerate(situations, 1): - lines.extend(self._render_situation(situation, i)) - lines.append("") - else: - lines.append("_No situations met the analysis threshold._") - lines.append("") - - if meta_fractures: - lines.append("## Meta-fractures") - lines.append("") - lines.append("_Underlying frame conflicts that surface across situations._") - lines.append("") - for mf in meta_fractures: - lines.append(f"### {mf.get('name', '(unnamed)')}") - if mf.get("description"): - lines.append("") - lines.append(mf["description"]) - indices = mf.get("situation_indices", []) - if indices: - sit_labels = ", ".join(f"Situation {i + 1}" for i in indices) - lines.append("") - lines.append(f"_Appears in: {sit_labels}_") - if mf.get("shared_point"): - lines.append("") - lines.append(f"**Shared point:** {mf['shared_point']}") - lines.append("") - - lines.extend(self._institutional_activity(metadata)) - - if thin_coverage: - lines.append("## Thin coverage") - lines.append("") - lines.append("_Topics with 1-2 articles. Listed but not fully analyzed._") - lines.append("") - for item in thin_coverage: - title = item.get("title", "Unknown topic") - count = item.get("article_count", 0) - sources = ", ".join(item.get("sources", [])) - note = item.get("note", "") - lines.append(f"- **{title}** — {count} article(s) | {sources}") - if note: - lines.append(f" - {note}") - lines.append("") - - return "\n".join(lines) - - def _institutional_activity(self, metadata: dict) -> list[str]: - """Institutional activity as markdown, or nothing when none was recorded.""" - block = institutional_activity(metadata) - moved, steady = split_activity(block) - if not moved and not steady: - return [] - - lines = ["## Institutional activity", "", f"_{ACTIVITY_NOTE}_", ""] - for entry in moved: - lines.append(f"- {activity_sentence(entry)}") - for entry in steady: - lines.append(f"- _{activity_sentence(entry)}_") - - footnote = activity_footnote(block) - if footnote: - lines.append("") - lines.append(f"_{footnote}_") - lines.append("") - return lines - - def _render_standing_agenda(self, doc: BriefDocument) -> list[str]: - """ - The beat's declared agenda, every item of it. - - Mirrors the terminal renderer exactly, including the part that matters: - an unmoved standing question is rendered with a NO MOVEMENT label, never - omitted. - """ - agenda = standing_agenda(doc.metadata) - if not agenda: - return [] - - lines = [ - "## Standing agenda", - "", - "_What this beat declared it is watching. Unmoved items are reported, not dropped._", - "", - ] - for entry in agenda: - lines.append(f"### [{standing_agenda_status(entry)}] {entry.get('text', '(question)')}") - lines.append("") - lines.append(f"_{standing_agenda_provenance(entry)}_") - lines.append("") - - movement = standing_agenda_movement(entry) - if movement: - for item in movement: - lines.append(f"- **Moved in:** {item}") - else: - lines.append(f"- {standing_agenda_no_movement(entry)}") - - for observable in entry.get("watching") or []: - lines.append(f"- **Watching for:** {observable}") - lines.append("") - - return lines - - def _render_situation(self, situation: dict, index: int) -> list[str]: - """Render a single situation as markdown.""" - lines: list[str] = [] - title = clean_citations(situation.get("title", "Untitled situation")) - lines.append(f"## Situation {index}: {title}") - lines.append("") - - narrative = situation.get("narrative", "") - if narrative: - for paragraph in clean_citations(narrative).split("\n\n"): - lines.append(paragraph.strip()) - lines.append("") - - actors = situation.get("actors", []) - if actors: - lines.append("### Actors") - lines.append("") - for actor in actors: - name = actor.get("name", "Unknown") - status = actor.get("epistemic_status", "") - status_tag = f" _[{status}]_" if status else "" - lines.append(f"- **{name}**{status_tag}") - if actor.get("role"): - lines.append(f" - Role: {actor['role']}") - if actor.get("interests"): - lines.append(f" - Interests: {actor['interests']}") - lines.append("") - - power = situation.get("power_dynamics", {}) - if power: - lines.append("### Power dynamics") - lines.append("") - if power.get("who_benefits"): - lines.append(f"- **Benefits:** {clean_citations(power['who_benefits'])}") - if power.get("who_is_harmed"): - lines.append(f"- **Harmed:** {clean_citations(power['who_is_harmed'])}") - if power.get("who_decides"): - lines.append(f"- **Decides:** {clean_citations(power['who_decides'])}") - lines.append("") - - frame = situation.get("coverage_frame", {}) - if frame: - lines.append("### What the coverage makes visible and invisible") - lines.append("") - if frame.get("narrative_layers"): - lines.append(f"- **Narratives:** {frame['narrative_layers']}") - elif frame.get("dominant_frame"): - lines.append(f"- **Frame:** {frame['dominant_frame']}") - if frame.get("fractures"): - lines.append(f"- **Fractures:** {frame['fractures']}") - if frame.get("bridges"): - lines.append(f"- **Bridges:** {frame['bridges']}") - if frame.get("structural_absences"): - lines.append(f"- **Hard to see:** {frame['structural_absences']}") - if frame.get("assumed_premise"): - lines.append(f"- **Assumes:** {frame['assumed_premise']}") - lines.append("") - - futures = situation.get("where_this_goes", {}) - if futures: - lines.append("### Where this goes") - lines.append("") - if futures.get("branching_paths"): - lines.append(f"- **Paths:** {clean_citations(futures['branching_paths'])}") - - primary_text, primary_prefix, secondary = question_lines(futures) - if primary_text: - prefix_part = f"_{primary_prefix}_ " if primary_prefix else "" - lines.append(f"- **Key question:** {prefix_part}{primary_text}") - for sec_text, sec_prefix in secondary: - prefix_part = f"_{sec_prefix}_ " if sec_prefix else "" - lines.append(f" - Also open: {prefix_part}{sec_text}") - - for item in watch_items(futures): - lines.append(f"- **Watch for:** {item}") - lines.append("") - - causal = situation.get("causal_structure", {}) - if causal: - lines.append("### Causal structure") - lines.append("") - if causal.get("forces"): - lines.append(f"- **Forces:** {clean_citations(causal['forces'])}") - if causal.get("constraints"): - lines.append(f"- **Constraints:** {clean_citations(causal['constraints'])}") - if causal.get("dependencies"): - lines.append(f"- **Dependencies:** {clean_citations(causal['dependencies'])}") - lines.append("") - - gaps = situation.get("information_gaps", []) - if gaps: - lines.append("### Information gaps") - lines.append("") - for gap in gaps: - missing = gap.get("what_is_missing", "") - lines.append(f"- **Gap:** {missing}") - if gap.get("why_it_matters"): - lines.append(f" - Why it matters: {gap['why_it_matters']}") - if gap.get("why_missing"): - lines.append(f" - Why missing: {gap['why_missing']}") - if gap.get("feed_recommendation"): - lines.append(f" - Suggested source: {gap['feed_recommendation']}") - lines.append("") - - return lines diff --git a/src/render/terminal.py b/src/render/terminal.py deleted file mode 100644 index 6d17d63..0000000 --- a/src/render/terminal.py +++ /dev/null @@ -1,374 +0,0 @@ -""" -Terminal renderer. - -The output of :meth:`TerminalRenderer.render` is the brief as InsightWeaver has -always printed it -- this is a move of the former ``BriefFormatter.format_report``, -not a rewrite. It is byte-for-byte compatible with the pre-split output for the -same synthesis payload, which is what makes replaying a stored run verifiable. -""" - -from __future__ import annotations - -from ..utils.base_formatter import BaseTerminalFormatter -from ..utils.colors import accent, header, muted, warning -from ._text import ( - ACTIVITY_NOTE, - DIRECTION_GLYPH, - activity_footnote, - activity_sentence, - clean_citations, - decision_summary, - institutional_activity, - prediction_check_line, - question_lines, - split_activity, - standing_agenda, - standing_agenda_movement, - standing_agenda_no_movement, - standing_agenda_provenance, - standing_agenda_status, - watch_items, -) -from .document import BriefDocument - -__all__ = ["TerminalRenderer", "clean_citations"] - - -class TerminalRenderer(BaseTerminalFormatter): - """Render a :class:`BriefDocument` for terminal display.""" - - def __init__(self, max_width: int = 80) -> None: - super().__init__(max_width) - - def render(self, doc: BriefDocument) -> str: - """Format a complete briefing for terminal output.""" - situations = doc.situations - thin_coverage = doc.thin_coverage - meta_fractures = doc.meta_fractures - metadata = doc.metadata - - lines = [] - - # Header - lines.append("") - lines.append(header("=" * self.max_width)) - lines.append(header("INTELLIGENCE BRIEF")) - lines.append(header("=" * self.max_width)) - lines.append("") - - # Metadata - articles = metadata.get("articles_analyzed", 0) - clusters_analyzed = metadata.get("clusters_analyzed", 0) - clusters_thin = metadata.get("clusters_thin", 0) - threshold = metadata.get("analysis_threshold", "3+ articles") - - lines.append( - muted( - f"Articles: {articles} | " - f"Situations: {clusters_analyzed} analyzed, {clusters_thin} thin coverage | " - f"Threshold: {threshold}" - ) - ) - - check_line = prediction_check_line(metadata.get("prediction_check")) - if check_line: - lines.append(muted(check_line)) - lines.append("") - - # Your decisions -- what today's coverage moved - decisions = decision_summary(metadata) - if decisions: - lines.append(header("YOUR DECISIONS")) - lines.append(muted("What today's coverage moved. See 'decisions show '.")) - lines.append("") - for entry in decisions: - lines.append(f" {accent(entry.get('decision', '(decision)'))}") - for factor in entry.get("factors", []): - direction = factor.get("direction", "neutral") - glyph = DIRECTION_GLYPH.get(direction, "~") - name = factor.get("name", "(factor)") - lines.append(f" [{glyph}] {name} {muted(f'({direction})')}") - lines.append("") - - # Standing agenda -- what this beat declared it is watching. - lines.extend(self._render_standing_agenda(doc)) - - # Situations - if situations: - for i, situation in enumerate(situations, 1): - lines.append(self._render_situation(situation, i)) - lines.append("") - else: - lines.append(muted("No situations met the analysis threshold.")) - lines.append("") - - # Meta-fractures across situations (Stage 6). - if meta_fractures: - lines.append(header("-" * self.max_width)) - lines.append(header("META-FRACTURES")) - lines.append(muted("Underlying frame conflicts that surface across situations.")) - lines.append(header("-" * self.max_width)) - lines.append("") - for mf in meta_fractures: - lines.append(accent(f" {mf.get('name', '(unnamed)')}")) - if mf.get("description"): - lines.append(f" {mf['description']}") - indices = mf.get("situation_indices", []) - if indices: - sit_labels = ", ".join(f"Situation {i + 1}" for i in indices) - lines.append(muted(f" Appears in: {sit_labels}")) - if mf.get("shared_point"): - lines.append(f" Shared point: {muted(mf['shared_point'])}") - lines.append("") - - # Institutional activity -- what the beat's declared institutions did - # this run against what they usually do. - lines.extend(self._render_institutional_activity(metadata)) - - # Thin coverage - if thin_coverage: - lines.append(header("-" * self.max_width)) - lines.append(header("THIN COVERAGE")) - lines.append(muted("Topics with 1-2 articles. Listed but not fully analyzed.")) - lines.append(header("-" * self.max_width)) - lines.append("") - - for item in thin_coverage: - title = item.get("title", "Unknown topic") - count = item.get("article_count", 0) - sources = ", ".join(item.get("sources", [])) - note = item.get("note", "") - - lines.append(f" {accent(title)}") - lines.append(f" {muted(f'{count} article(s) | {sources}')}") - if note: - lines.append(f" {note}") - lines.append("") - - lines.append(header("=" * self.max_width)) - - return "\n".join(lines) - - def _render_institutional_activity(self, metadata: dict) -> list[str]: - """ - Format the institutional activity section, or nothing at all. - - Two blocks, and both matter. The first is what moved. The second is the - entities that have been active before and are steady or silent today -- - quieter, but present, because an office going quiet is information and - a section that only ever showed increases would read as a scoreboard. - """ - block = institutional_activity(metadata) - moved, steady = split_activity(block) - if not moved and not steady: - return [] - - lines = [ - header("-" * self.max_width), - header("INSTITUTIONAL ACTIVITY"), - muted(ACTIVITY_NOTE), - header("-" * self.max_width), - "", - ] - for entry in moved: - lines.append(f" {activity_sentence(entry)}") - if moved and steady: - lines.append("") - for entry in steady: - lines.append(muted(f" {activity_sentence(entry)}")) - - footnote = activity_footnote(block) - if footnote: - lines.append("") - lines.append(muted(f" {footnote}")) - lines.append("") - return lines - - def _render_standing_agenda(self, doc: BriefDocument) -> list[str]: - """ - The beat's declared agenda, every item of it. - - Each entry is rendered whether or not it moved. There is deliberately - no "skip the quiet ones" branch here: a standing question that vanishes - on a quiet day is the exact failure this section exists to prevent, so - an unmoved question gets a NO MOVEMENT label and a sentence saying so. - """ - agenda = standing_agenda(doc.metadata) - if not agenda: - return [] - - lines = [ - header("STANDING AGENDA"), - muted( - "What this beat declared it is watching. Unmoved items are reported, not dropped." - ), - "", - ] - for entry in agenda: - status = standing_agenda_status(entry) - label = accent(f"[{status}]") if entry.get("moved") else warning(f"[{status}]") - lines.append(f" {label} {entry.get('text', '(question)')}") - lines.append(f" {muted(standing_agenda_provenance(entry))}") - - movement = standing_agenda_movement(entry) - if movement: - for item in movement: - lines.append(f" Moved in: {item}") - else: - lines.append(f" {muted(standing_agenda_no_movement(entry))}") - - for observable in entry.get("watching") or []: - lines.append(f" Watching for: {muted(str(observable))}") - lines.append("") - - return lines - - def _render_situation(self, situation: dict, index: int) -> str: - """Format a single situation for terminal display.""" - lines = [] - - title = clean_citations(situation.get("title", "Untitled situation")) - lines.append(header(f"SITUATION {index}: {title}")) - lines.append(header("-" * self.max_width)) - - # Narrative - narrative = situation.get("narrative", "") - if narrative: - lines.append("") - for paragraph in clean_citations(narrative).split("\n\n"): - lines.append(self.wrap_text(paragraph.strip())) - lines.append("") - - # Actors - actors = situation.get("actors", []) - if actors: - lines.append(accent(" ACTORS:")) - for actor in actors: - name = actor.get("name", "Unknown") - role = actor.get("role", "") - interests = actor.get("interests", "") - status = actor.get("epistemic_status", "") - status_tag = f" [{status}]" if status else "" - - lines.append(f" {name}{muted(status_tag)}") - if role: - lines.append(f" Role: {role}") - if interests: - lines.append(f" Interests: {interests}") - lines.append("") - - # Power dynamics - power = situation.get("power_dynamics", {}) - if power: - lines.append(accent(" POWER DYNAMICS:")) - if power.get("who_benefits"): - lines.append(f" Benefits: {clean_citations(power['who_benefits'])}") - if power.get("who_is_harmed"): - lines.append(f" Harmed: {clean_citations(power['who_is_harmed'])}") - if power.get("who_decides"): - lines.append(f" Decides: {clean_citations(power['who_decides'])}") - lines.append("") - - # Coverage frame / narrative layers - frame = situation.get("coverage_frame", {}) - if frame: - lines.append(accent(" WHAT THE COVERAGE MAKES VISIBLE AND INVISIBLE:")) - if frame.get("narrative_layers"): - lines.append(f" Narratives: {frame['narrative_layers']}") - elif frame.get("dominant_frame"): - # Backward compat with old schema - lines.append(f" Frame: {frame['dominant_frame']}") - if frame.get("fractures"): - lines.append(f" Fractures: {frame['fractures']}") - if frame.get("bridges"): - lines.append(f" Bridges: {muted(frame['bridges'])}") - if frame.get("structural_absences"): - lines.append(f" Hard to see: {muted(frame['structural_absences'])}") - if frame.get("assumed_premise"): - lines.append(f" Assumes: {muted(frame['assumed_premise'])}") - lines.append("") - - # Where this goes - futures = situation.get("where_this_goes", {}) - if futures: - lines.append(accent(" WHERE THIS GOES:")) - if futures.get("branching_paths"): - lines.append(f" Paths: {clean_citations(futures['branching_paths'])}") - - primary_text, primary_prefix, secondary = question_lines(futures) - if primary_text: - tag = f"{muted(primary_prefix)} " if primary_prefix else "" - lines.append(f" Key question: {tag}{primary_text}") - for sec_text, sec_prefix in secondary: - tag = f"{muted(sec_prefix)} " if sec_prefix else "" - lines.append(f" Also open: {tag}{muted(sec_text)}") - - for item in watch_items(futures): - lines.append(f" Watch for: {muted(item)}") - lines.append("") - - # Causal structure (backward compat -- may not be present in new schema) - causal = situation.get("causal_structure", {}) - if causal: - lines.append(accent(" CAUSAL STRUCTURE:")) - if causal.get("forces"): - lines.append(f" Forces: {clean_citations(causal['forces'])}") - if causal.get("constraints"): - lines.append(f" Constraints: {clean_citations(causal['constraints'])}") - if causal.get("dependencies"): - lines.append(f" Dependencies: {clean_citations(causal['dependencies'])}") - lines.append("") - - # Information gaps - gaps = situation.get("information_gaps", []) - if gaps: - lines.append(accent(" INFORMATION GAPS:")) - for gap in gaps: - missing = gap.get("what_is_missing", "") - why = gap.get("why_it_matters", "") - why_missing = gap.get("why_missing", "") - feed = gap.get("feed_recommendation", "") - - lines.append(f" {warning('GAP')}: {missing}") - if why: - lines.append(f" Why it matters: {muted(why)}") - if why_missing: - lines.append(f" Why missing: {muted(why_missing)}") - if feed: - lines.append(f" Suggested source: {muted(feed)}") - lines.append("") - - return "\n".join(lines) - - def render_compact(self, doc: BriefDocument) -> str: - """ - Situation titles only, for scripts and automation (``--quiet``). - - Emitted as one string rather than a sequence of ``click.echo`` calls so - that there is a single place that decides this layout. - """ - lines: list[str] = [""] - - if doc.situations: - lines.append(header("Situations analyzed:")) - for i, situation in enumerate(doc.situations, 1): - title = clean_citations(situation.get("title", "Untitled")) - lines.append(f" {accent(f'{i}.')} {title}") - - if doc.thin_coverage: - lines.append(muted(f"\n+ {len(doc.thin_coverage)} topics with thin coverage")) - - lines.append( - muted( - f"\nArticles: {doc.articles_analyzed} | " - f"Clusters: {doc.metadata.get('clusters_total', 0)}" - ) - ) - return "\n".join(lines) - - def render_one_line_summary(self, doc: BriefDocument) -> str: - """One-line summary for the command refresher.""" - return ( - f"BRIEF: {doc.metadata.get('articles_analyzed', 0)} articles, " - f"{len(doc.situations)} situations analyzed" - ) diff --git a/src/utils/base_formatter.py b/src/utils/base_formatter.py deleted file mode 100644 index f82905c..0000000 --- a/src/utils/base_formatter.py +++ /dev/null @@ -1,192 +0,0 @@ -""" -Base Terminal Formatter -Common formatting utilities for terminal output -""" - -import textwrap -from typing import Any - - -class BaseTerminalFormatter: - """Base class with common terminal formatting utilities""" - - def __init__(self, max_width: int = 80): - self.max_width = max_width - - def format_header(self, title: str, char: str = "=") -> str: - """ - Format a section header with decorative characters - - Args: - title: Header title text - char: Character to use for decoration (default: '=') - - Returns: - Formatted header string with decoration lines - """ - return f"{char * self.max_width}\n{title}\n{char * self.max_width}" - - def format_subheader(self, title: str, char: str = "-") -> str: - """ - Format a subsection header - - Args: - title: Subheader title text - char: Character to use for decoration (default: '-') - - Returns: - Formatted subheader string - """ - return f"{char * self.max_width}\n{title}\n{char * self.max_width}" - - def wrap_text( - self, - text: str, - indent: int = 0, - subsequent_indent: int | None = None, - ) -> str: - """ - Wrap text to fit within max_width - - Args: - text: Text to wrap - indent: Number of spaces for initial indent - subsequent_indent: Indent for subsequent lines (defaults to indent) - - Returns: - Wrapped text string - """ - initial = " " * indent - subsequent = " " * (subsequent_indent if subsequent_indent is not None else indent) - return textwrap.fill( - text, - width=self.max_width, - initial_indent=initial, - subsequent_indent=subsequent, - ) - - def format_list( - self, - items: list[str], - bullet: str = "-", - indent: int = 2, - ) -> str: - """ - Format a list of items with bullets - - Args: - items: List of string items - bullet: Bullet character (default: '-') - indent: Indentation level - - Returns: - Formatted list string - """ - lines = [] - prefix = " " * indent - bullet_prefix = f"{prefix}{bullet} " - wrap_indent = " " * (indent + len(bullet) + 1) - - for item in items: - wrapped = textwrap.fill( - item, - width=self.max_width, - initial_indent=bullet_prefix, - subsequent_indent=wrap_indent, - ) - lines.append(wrapped) - - return "\n".join(lines) - - def format_numbered_list( - self, - items: list[str], - indent: int = 0, - start: int = 1, - ) -> str: - """ - Format a numbered list of items - - Args: - items: List of string items - indent: Indentation level - start: Starting number - - Returns: - Formatted numbered list string - """ - lines = [] - prefix = " " * indent - - for i, item in enumerate(items, start): - number_prefix = f"{prefix}{i}. " - wrap_indent = " " * (indent + len(str(i)) + 2) - wrapped = textwrap.fill( - item, - width=self.max_width, - initial_indent=number_prefix, - subsequent_indent=wrap_indent, - ) - lines.append(wrapped) - - return "\n".join(lines) - - def format_key_value( - self, - key: str, - value: Any, - separator: str = ": ", - indent: int = 0, - ) -> str: - """ - Format a key-value pair - - Args: - key: The key/label - value: The value (will be converted to string) - separator: Separator between key and value - indent: Indentation level - - Returns: - Formatted key-value string - """ - prefix = " " * indent - key_prefix = f"{prefix}{key}{separator}" - wrap_indent = " " * len(key_prefix) - - return textwrap.fill( - str(value), - width=self.max_width, - initial_indent=key_prefix, - subsequent_indent=wrap_indent, - ) - - def format_section( - self, - title: str, - content: str | list[str], - header_char: str = "-", - ) -> str: - """ - Format a complete section with header and content - - Args: - title: Section title - content: Section content (string or list of lines) - header_char: Character for header decoration - - Returns: - Formatted section string - """ - lines = [ - self.format_subheader(title, header_char), - "", - ] - - if isinstance(content, list): - lines.extend(content) - else: - lines.append(content) - - lines.append("") - return "\n".join(lines) diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index 6377014..a8537d6 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -1,8 +1,10 @@ """ -CLI-specific test fixtures -""" +CLI-specific test fixtures. -from unittest.mock import MagicMock +The ``mock_api_key`` / ``mock_no_api_key`` fixtures were removed by backlog +task 012: both patched ``src.cli.brief.settings``, and the brief command is +gone. Nothing surviving in the CLI reads an API key. +""" import pytest from click.testing import CliRunner @@ -12,19 +14,3 @@ def cli_runner(): """Click CLI test runner.""" return CliRunner() - - -@pytest.fixture -def mock_api_key(monkeypatch): - """Mock API key as configured for the brief command.""" - mock = MagicMock() - mock.anthropic_api_key = "test-api-key" - monkeypatch.setattr("src.cli.brief.settings", mock) - - -@pytest.fixture -def mock_no_api_key(monkeypatch): - """Mock API key as not configured.""" - mock = MagicMock() - mock.anthropic_api_key = None - monkeypatch.setattr("src.cli.brief.settings", mock) diff --git a/tests/cli/test_app.py b/tests/cli/test_app.py index 6f81c97..b6d6de3 100644 --- a/tests/cli/test_app.py +++ b/tests/cli/test_app.py @@ -4,6 +4,8 @@ from unittest.mock import patch +import pytest + from src.cli.app import cli, interactive_mode, print_command_refresher @@ -16,7 +18,7 @@ def test_cli_help(self, cli_runner): assert result.exit_code == 0 assert "InsightWeaver" in result.output - assert "RSS Feed Analysis" in result.output + assert "RSS feed ingestion" in result.output def test_cli_version(self, cli_runner): """Should show version""" @@ -44,26 +46,36 @@ def test_print_command_refresher(self, cli_runner, capsys): captured = capsys.readouterr() assert "Commands:" in captured.out - assert "brief" in captured.out - assert "forecast" in captured.out + assert "sources" in captured.out class TestSubcommandRegistration: """Tests for subcommand registration""" - def test_brief_command_registered(self, cli_runner): - """Should have brief command registered""" - result = cli_runner.invoke(cli, ["brief", "--help"]) - - assert result.exit_code == 0 - assert "hours" in result.output.lower() - - def test_forecast_command_registered(self, cli_runner): - """Should have forecast command registered""" - result = cli_runner.invoke(cli, ["forecast", "--help"]) + def test_sources_command_registered(self, cli_runner): + """Should have sources command registered""" + result = cli_runner.invoke(cli, ["sources", "--help"]) assert result.exit_code == 0 - assert "forecast" in result.output.lower() + assert "list" in result.output.lower() + + @pytest.mark.parametrize( + "gone", + ["brief", "frames", "diet", "questions", "predictions", "forecast", "decisions", "beat"], + ) + def test_deleted_commands_are_not_registered(self, cli_runner, gone): + """ + The briefing commands are gone from --help and from dispatch. + + Pinned rather than assumed: the editable install resolves a missing + ``src.*`` module against the developer's other checkout, so a dangling + command import can appear to work locally. This asserts on the + registered command table instead. + """ + assert gone not in cli.commands + + result = cli_runner.invoke(cli, [gone, "--help"]) + assert result.exit_code != 0 class TestAsciiArt: diff --git a/tests/cli/test_beat_coverage.py b/tests/cli/test_beat_coverage.py deleted file mode 100644 index 871a77a..0000000 --- a/tests/cli/test_beat_coverage.py +++ /dev/null @@ -1,304 +0,0 @@ -""" -Tests for ``insightweaver beat coverage NAME``. - -What is pinned here is the gate: the exit code, and the fact that the counts -printed under it always add back up to the number of probes the beat declared. -The command is only useful if a shell can trust it, and the two ways to break -that trust are exiting 0 on a miss and exiting 0 on a run that measured nothing. - -Every beat file and every article here is written by the test. No test names a -real event or reads the shipped beat's probes -- those are statements about live -corpus state, and pinning them into CI would make this suite fail for reasons -unrelated to the code under test. - -Also pinned: the command runs with no ANTHROPIC_API_KEY and never writes. -""" - -import json -from datetime import datetime - -import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -from src.cli.beat import ( - EXIT_NOTHING_MEASURED, - EXIT_OK, - EXIT_UNMATCHED, - beat_command, -) -from src.config.beats import load_beat -from src.config.feed_matcher import Feed -from src.database.models import Article, Base, RSSFeed - -FEED_URL = "https://example.test/in-beat" -FEED_NAME = "In-Beat Wire" - -MATCHING_PROBE = { - "date": "2026-08-24", - "what": "the thing that happened", - "terms": ["FedRAMP"], - "any_of": [["director", "administrator"], ["reinstat*", "restored"]], -} -MISSING_PROBE = { - "date": "2026-08-24", - "what": "the thing nobody carried", - "terms": ["CMMC"], - "any_of": [["phase 2", "phase two"]], -} -OLD_PROBE = { - "date": "2019-01-15", - "what": "the thing that predates the corpus", - "terms": ["CISA"], - "any_of": [["binding operational directive"]], -} - - -@pytest.fixture -def beats_dir(tmp_path, monkeypatch): - """A beats directory the test owns, wired into the command.""" - directory = tmp_path / "beats" - directory.mkdir() - monkeypatch.setattr( - "src.cli.beat.load_beat", lambda name, beats_dir=directory: load_beat(name, beats_dir) - ) - return directory - - -@pytest.fixture -def write_beat(beats_dir): - def _write(probes, name="probe-beat"): - (beats_dir / f"{name}.json").write_text( - json.dumps( - { - "name": name, - "description": "A beat under test.", - "sources": [{"adapter": "rss", "feed_tags": ["regulatory"]}], - "coverage_probes": probes, - } - ) - ) - return name - - return _write - - -@pytest.fixture -def fake_feeds(monkeypatch): - """One configured feed, so the beat resolves without reading config/feeds.""" - - class FakeMatcher: - all_feeds = [ - Feed( - name=FEED_NAME, - url=FEED_URL, - scope=[], - geo_tags=[], - domain_tags=["regulatory"], - specialty_tags=[], - relevance_score=0.9, - source_file="fake.json", - ) - ] - - monkeypatch.setattr("src.config.beats.FeedMatcher", FakeMatcher) - - -@pytest.fixture -def corpus(tmp_path, monkeypatch): - """ - A throwaway database wired into the command, plus a helper to fill it. - - ``SessionLocal`` is replaced rather than the global engine so that no test - here depends on whatever DATABASE_URL happens to name. - """ - engine = create_engine(f"sqlite:///{tmp_path / 'coverage.db'}") - Base.metadata.create_all(bind=engine) - maker = sessionmaker(bind=engine) - monkeypatch.setattr("src.cli.beat.SessionLocal", maker) - - session = maker() - feed = RSSFeed(url=FEED_URL, name=FEED_NAME, category="test") - session.add(feed) - session.commit() - - def add(title, when=datetime(2026, 8, 25)): - session.add( - Article( - feed_id=feed.id, - guid=f"guid-{title}", - title=title, - description="", - normalized_content="", - published_date=when, - ) - ) - session.commit() - - add.session = session - add.engine = engine - yield add - session.close() - - -@pytest.fixture(autouse=True) -def no_api_key(monkeypatch): - """The command must work with no key at all; this is not optional.""" - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - - -def invoke(cli_runner, name): - return cli_runner.invoke(beat_command, ["coverage", name]) - - -class TestTheGate: - def test_all_probes_matched_exits_zero(self, cli_runner, write_beat, fake_feeds, corpus): - corpus("FedRAMP director reinstated after review") - result = invoke(cli_runner, write_beat([MATCHING_PROBE])) - - assert result.exit_code == EXIT_OK - assert "[MATCHED]" in result.output - assert "PASS" in result.output - - def test_an_unmatched_probe_exits_non_zero(self, cli_runner, write_beat, fake_feeds, corpus): - corpus("An unrelated procurement notice") - result = invoke(cli_runner, write_beat([MISSING_PROBE])) - - assert result.exit_code == EXIT_UNMATCHED - assert result.exit_code != 0 - assert "[UNMATCHED]" in result.output - assert "FAIL" in result.output - - def test_one_miss_among_matches_still_fails(self, cli_runner, write_beat, fake_feeds, corpus): - corpus("FedRAMP director reinstated after review") - result = invoke(cli_runner, write_beat([MATCHING_PROBE, MISSING_PROBE])) - - assert result.exit_code == EXIT_UNMATCHED - assert "[MATCHED]" in result.output - assert "[UNMATCHED]" in result.output - - def test_a_beat_with_no_probes_is_not_a_pass(self, cli_runner, write_beat, fake_feeds, corpus): - """ - An undeclared probe set is the state the beat was in when it missed the - event this feature exists because of. It must not exit 0. - """ - result = invoke(cli_runner, write_beat([])) - - assert result.exit_code == EXIT_NOTHING_MEASURED - assert "declares no coverage_probes" in result.output - - def test_every_probe_inconclusive_is_not_a_pass_either( - self, cli_runner, write_beat, fake_feeds, corpus - ): - corpus("Recent unrelated news") - result = invoke(cli_runner, write_beat([OLD_PROBE])) - - assert result.exit_code == EXIT_NOTHING_MEASURED - assert "[INCONCLUSIVE]" in result.output - assert "NOT MEASURED" in result.output - - def test_the_three_exit_codes_are_distinct(self): - assert len({EXIT_OK, EXIT_UNMATCHED, EXIT_NOTHING_MEASURED}) == 3 - - -class TestOutput: - def test_a_match_names_the_feed_that_carried_it( - self, cli_runner, write_beat, fake_feeds, corpus - ): - corpus("FedRAMP director reinstated after review") - result = invoke(cli_runner, write_beat([MATCHING_PROBE])) - assert FEED_NAME in result.output - - def test_an_unmatched_probe_says_what_the_closest_article_lacked( - self, cli_runner, write_beat, fake_feeds, corpus - ): - corpus("CMMC rulemaking continues") - result = invoke(cli_runner, write_beat([MISSING_PROBE])) - assert "phase 2" in result.output - - def test_an_inconclusive_probe_says_why(self, cli_runner, write_beat, fake_feeds, corpus): - corpus("Recent unrelated news") - result = invoke(cli_runner, write_beat([OLD_PROBE])) - assert "predates the corpus" in result.output - - def test_the_denominator_is_every_probe_declared( - self, cli_runner, write_beat, fake_feeds, corpus - ): - """ - The inconclusive probe stays in the count. A suite that decayed to only - the probes still matching would print '1 matched ... of 1 declared'. - """ - corpus("FedRAMP director reinstated after review") - result = invoke(cli_runner, write_beat([MATCHING_PROBE, OLD_PROBE])) - - assert "1 matched + 0 unmatched + 1 inconclusive of 2 probe(s) declared" in result.output - assert result.exit_code == EXIT_OK - assert "counted above, not dropped" in result.output - - def test_show_problems_hides_the_passes_only(self, cli_runner, write_beat, fake_feeds, corpus): - corpus("FedRAMP director reinstated after review") - result = cli_runner.invoke( - beat_command, - ["coverage", write_beat([MATCHING_PROBE, OLD_PROBE]), "--show", "problems"], - ) - assert "the thing that happened" not in result.output - assert "the thing that predates the corpus" in result.output - assert "of 2 probe(s) declared" in result.output - - -class TestFailures: - def test_an_unknown_beat_is_an_error_not_a_pass(self, cli_runner, beats_dir, corpus): - result = invoke(cli_runner, "no-such-beat") - assert result.exit_code != 0 - assert "No beat named" in result.output - - def test_an_invalid_probe_block_is_reported_by_name(self, cli_runner, beats_dir, corpus): - (beats_dir / "broken.json").write_text( - json.dumps( - { - "name": "broken", - "sources": [{"adapter": "rss", "feed_tags": ["regulatory"]}], - "coverage_probes": [{"date": "2026-08-24", "what": "x", "terms": ["FedRAMP"]}], - } - ) - ) - result = invoke(cli_runner, "broken") - assert result.exit_code != 0 - assert "single term" in result.output - - -class TestItNeverWrites: - def test_the_corpus_is_unchanged_by_a_run(self, cli_runner, write_beat, fake_feeds, corpus): - corpus("FedRAMP director reinstated after review") - corpus("An unrelated procurement notice") - before = corpus.session.query(Article).count() - - invoke(cli_runner, write_beat([MATCHING_PROBE, MISSING_PROBE, OLD_PROBE])) - - corpus.session.expire_all() - assert corpus.session.query(Article).count() == before - - def test_the_command_never_commits( - self, cli_runner, write_beat, fake_feeds, corpus, monkeypatch - ): - """ - The corpus is shared. Committing nothing is still an intent to write, so - the read path must not reach for commit at all. - """ - maker = sessionmaker(bind=corpus.engine) - commits = [] - - class Watched(maker.class_): - def commit(self, *args, **kwargs): - commits.append(True) - raise AssertionError("beat coverage must not commit") - - monkeypatch.setattr( - "src.cli.beat.SessionLocal", sessionmaker(bind=corpus.engine, class_=Watched) - ) - corpus("FedRAMP director reinstated after review") - - result = invoke(cli_runner, write_beat([MATCHING_PROBE])) - - assert commits == [] - assert result.exit_code == EXIT_OK diff --git a/tests/cli/test_brief_beat.py b/tests/cli/test_brief_beat.py deleted file mode 100644 index 7d67ea1..0000000 --- a/tests/cli/test_brief_beat.py +++ /dev/null @@ -1,245 +0,0 @@ -""" -Tests for `insightweaver brief --beat`. - -Three things are pinned here: - -* the beat reaches the pipeline, and only after being loaded and validated; -* a bad ``--beat`` costs nothing -- the pipeline is never started; -* **the no-beat default path is unchanged** -- ``brief`` with no ``--beat`` - still calls ``run_pipeline`` with exactly the arguments it did before, plus - an explicit ``beat=None``, and the render-only ``--from-run`` path is - untouched. -""" - -import json -from unittest.mock import MagicMock - -import pytest - -from src.cli.brief import brief_group - - -@pytest.fixture -def api_key(monkeypatch): - settings = MagicMock() - settings.anthropic_api_key = "test-api-key" - monkeypatch.setattr("src.cli.brief.settings", settings) - return settings - - -@pytest.fixture -def pipeline_spy(monkeypatch): - """Capture the run_pipeline call and return a run that found nothing.""" - calls = [] - - async def _spy(**kwargs): - calls.append(kwargs) - return { - "stages": {"synthesis": {"status": "no_articles", "articles_analyzed": 0}}, - "summary": {"duration_seconds": 0.0, "articles_fetched": 0}, - } - - monkeypatch.setattr("src.cli.brief.run_pipeline", _spy) - return calls - - -@pytest.fixture -def beat_ready_engine(tmp_path, monkeypatch): - """ - A throwaway database that has already run the beats migration. - - Bound explicitly so no test in this module depends on whatever - DATABASE_URL happens to name. - """ - from sqlalchemy import create_engine - - from src.database.models import Base - - engine = create_engine(f"sqlite:///{tmp_path / 'beat-ready.db'}") - Base.metadata.create_all(bind=engine) - monkeypatch.setattr("src.cli.brief.engine", engine) - return engine - - -@pytest.fixture -def pipeline_fuse(monkeypatch): - """Blow up if the pipeline is reached at all.""" - - async def _fuse(*args, **kwargs): - raise AssertionError("the pipeline must not start") - - monkeypatch.setattr("src.cli.brief.run_pipeline", _fuse) - - -@pytest.fixture -def beat_on_disk(tmp_path, monkeypatch): - """A valid beat in a throwaway beats directory.""" - directory = tmp_path / "beats" - directory.mkdir() - (directory / "test-beat.json").write_text( - json.dumps( - { - "name": "test-beat", - "description": "A test beat.", - "sources": [{"adapter": "rss", "feed_tags": ["regulatory"]}], - "coverage": {}, - "standing_questions": [], - "channels": ["terminal"], - } - ) - ) - monkeypatch.setattr("src.config.beats.DEFAULT_BEATS_DIR", directory) - return directory - - -@pytest.fixture -def malformed_beat_on_disk(tmp_path, monkeypatch): - directory = tmp_path / "beats" - directory.mkdir() - (directory / "broken-beat.json").write_text("{ this is not json") - monkeypatch.setattr("src.config.beats.DEFAULT_BEATS_DIR", directory) - return directory - - -class TestNoBeatIsTheDefaultPath: - def test_run_pipeline_called_exactly_as_before(self, cli_runner, api_key, pipeline_spy): - result = cli_runner.invoke(brief_group, [], obj={}) - - assert result.exit_code == 0, result.output - assert pipeline_spy == [{"prioritize_hours": 24, "topic_filters": {}, "beat": None}] - - def test_topic_filters_still_reach_the_pipeline(self, cli_runner, api_key, pipeline_spy): - result = cli_runner.invoke(brief_group, ["-cs", "-n", "--hours", "48"], obj={}) - - assert result.exit_code == 0, result.output - assert pipeline_spy == [ - { - "prioritize_hours": 48, - "topic_filters": {"topics": ["cybersecurity"], "scopes": ["national"]}, - "beat": None, - } - ] - - def test_no_beat_never_touches_the_beats_directory( - self, cli_runner, api_key, pipeline_spy, monkeypatch - ): - """The default brief must not depend on config/beats/ existing at all.""" - - def _explode(*args, **kwargs): - raise AssertionError("the default path must not load a beat") - - monkeypatch.setattr("src.cli.brief.load_beat", _explode) - - result = cli_runner.invoke(brief_group, [], obj={}) - - assert result.exit_code == 0, result.output - - -class TestBeatReachesThePipeline: - def test_loaded_beat_is_passed_through( - self, cli_runner, api_key, pipeline_spy, beat_on_disk, beat_ready_engine - ): - result = cli_runner.invoke(brief_group, ["--beat", "test-beat"], obj={}) - - assert result.exit_code == 0, result.output - assert len(pipeline_spy) == 1 - beat = pipeline_spy[0]["beat"] - assert beat is not None - assert beat.name == "test-beat" - assert beat.sources[0].feed_tags == ("regulatory",) - - def test_empty_beat_is_reported_rather_than_rendered_blank( - self, cli_runner, api_key, pipeline_spy, beat_on_disk, beat_ready_engine - ): - """A beat with nothing in it is an expected outcome, and it says so.""" - result = cli_runner.invoke(brief_group, ["--beat", "test-beat"], obj={}) - - assert result.exit_code == 0, result.output - assert "NO ARTICLES IN BEAT 'test-beat'" in result.output - - -class TestBadBeatCostsNothing: - def test_unknown_beat_names_what_is_available( - self, cli_runner, api_key, pipeline_fuse, beat_on_disk, beat_ready_engine - ): - result = cli_runner.invoke(brief_group, ["--beat", "nope"], obj={}) - - assert result.exit_code != 0 - assert "No beat named 'nope'" in result.output - assert "test-beat" in result.output - - def test_malformed_beat_names_the_problem( - self, cli_runner, api_key, pipeline_fuse, malformed_beat_on_disk, beat_ready_engine - ): - result = cli_runner.invoke(brief_group, ["--beat", "broken-beat"], obj={}) - - assert result.exit_code != 0 - assert "not valid JSON" in result.output - - def test_beat_is_resolved_before_the_api_key_matters( - self, cli_runner, pipeline_fuse, beat_on_disk, beat_ready_engine, monkeypatch - ): - """A typo should not be masked by an unrelated missing-key abort.""" - settings = MagicMock() - settings.anthropic_api_key = "" - monkeypatch.setattr("src.cli.brief.settings", settings) - - result = cli_runner.invoke(brief_group, ["--beat", "nope"], obj={}) - - assert result.exit_code != 0 - assert "No beat named 'nope'" in result.output - - -class TestBeatAndFromRunAreMutuallyExclusive: - def test_rejected_before_loading_anything(self, cli_runner, monkeypatch, pipeline_fuse): - def _explode(_id): - raise AssertionError("--from-run must not load a brief when --beat is given") - - monkeypatch.setattr("src.cli.brief.load_stored_brief", _explode) - - result = cli_runner.invoke( - brief_group, ["--from-run", "176", "--beat", "test-beat"], obj={} - ) - - assert result.exit_code != 0 - assert "--beat applies to the live pipeline" in result.output - - -class TestUnmigratedDatabaseIsRefusedNotMisreported: - """ - A beat run on a database without the beat tables must say so, not run and - then report "no articles" -- the wrong reason is worse than no reason. - """ - - @pytest.fixture - def pre_beat_engine(self, tmp_path, monkeypatch): - from sqlalchemy import create_engine - - from src.database.models import Base - - engine = create_engine(f"sqlite:///{tmp_path / 'pre-beat.db'}") - Base.metadata.create_all( - bind=engine, - tables=[ - table - for name, table in Base.metadata.tables.items() - if name not in ("beats", "beat_runs") - ], - ) - monkeypatch.setattr("src.cli.brief.engine", engine) - return engine - - def test_beat_run_is_refused_with_the_migration_command( - self, cli_runner, api_key, pipeline_fuse, beat_on_disk, pre_beat_engine - ): - result = cli_runner.invoke(brief_group, ["--beat", "test-beat"], obj={}) - - assert result.exit_code != 0 - assert "add_beats" in result.output - assert "NO ARTICLES" not in result.output - - def test_default_brief_is_unaffected(self, cli_runner, api_key, pipeline_spy, pre_beat_engine): - result = cli_runner.invoke(brief_group, [], obj={}) - - assert result.exit_code == 0, result.output - assert pipeline_spy == [{"prioritize_hours": 24, "topic_filters": {}, "beat": None}] diff --git a/tests/cli/test_brief_formatter.py b/tests/cli/test_brief_formatter.py deleted file mode 100644 index 0a2ea3d..0000000 --- a/tests/cli/test_brief_formatter.py +++ /dev/null @@ -1,350 +0,0 @@ -""" -Tests for brief_formatter — focused on the question-identity render integration. -""" - -import re - -from src.cli.brief_formatter import BriefFormatter - -# The formatting helpers moved to src/render/_text.py when brief rendering was -# split out; BriefFormatter is now a thin adapter over the renderers. These -# tests still exercise the same behaviour through the same public entry point. -from src.render._text import ( - decision_summary as _decision_summary, -) -from src.render._text import ( - prediction_check_line as _prediction_check_line, -) -from src.render._text import ( - question_lines as _question_lines, -) -from src.render._text import ( - watch_items as _watch_items, -) - - -def strip_ansi(text: str) -> str: - """Drop ANSI color escapes so assertions can match on raw content.""" - return re.sub(r"\x1b\[[0-9;]*m", "", text) - - -class TestQuestionLines: - def test_returns_empty_when_no_questions(self): - assert _question_lines({}) == ("", "", []) - - def test_legacy_string_unresolved_question(self): - text, prefix, secondary = _question_lines({"unresolved_question": "Will A?"}) - assert text == "Will A?" - assert prefix == "" - assert secondary == [] - - def test_new_shape_first_run_has_new_prefix(self): - out = _question_lines( - { - "unresolved_questions": { - "primary": { - "text": "Will A?", - "question_id": 7, - "first_asked_at": "2026-05-12T10:00:00", - "appearance_count": 1, - }, - "secondary": [], - } - } - ) - assert out[0] == "Will A?" - assert out[1] == "Q7 (new)" - assert out[2] == [] - - def test_new_shape_returning_question_shows_run_and_date(self): - out = _question_lines( - { - "unresolved_questions": { - "primary": { - "text": "Will A?", - "question_id": 7, - "first_asked_at": "2026-03-12T10:00:00", - "appearance_count": 4, - }, - "secondary": [ - { - "text": "Will B?", - "question_id": 8, - "first_asked_at": "2026-05-01T08:00:00", - "appearance_count": 2, - } - ], - } - } - ) - assert out[0] == "Will A?" - assert out[1] == "Q7 (run 4, asked 2026-03-12)" - assert out[2] == [("Will B?", "Q8 (run 2, asked 2026-05-01)")] - - -class TestPredictionCheckLine: - def test_empty_when_no_check(self): - assert _prediction_check_line(None) == "" - assert _prediction_check_line({}) == "" - - def test_empty_when_nothing_checked_or_expired(self): - assert _prediction_check_line({"checked": 0, "expired": 0}) == "" - - def test_renders_summary(self): - line = _prediction_check_line( - { - "checked": 10, - "triggered": 2, - "contradicted": 1, - "still_open": 7, - "expired": 3, - } - ) - assert "10 open observables graded" in line - assert "2 triggered" in line - assert "1 contradicted" in line - assert "7 still open" in line - assert "3 expired" in line - - def test_renders_when_only_expired(self): - line = _prediction_check_line({"checked": 0, "expired": 4}) - assert "4 expired" in line - - -class TestWatchItems: - def test_list_of_objects(self): - futures = { - "what_to_watch": [ - {"observable": "Fed cuts", "trigger_condition": "cut announced"}, - {"observable": "Yield inverts", "trigger_condition": "2y > 10y"}, - ] - } - assert _watch_items(futures) == [ - "Fed cuts -- cut announced", - "Yield inverts -- 2y > 10y", - ] - - def test_legacy_string(self): - assert _watch_items({"what_to_watch": "Watch the Fed"}) == ["Watch the Fed"] - - def test_observable_only(self): - assert _watch_items({"what_to_watch": [{"observable": "Just this"}]}) == ["Just this"] - - def test_empty(self): - assert _watch_items({}) == [] - assert _watch_items({"what_to_watch": []}) == [] - - -class TestMetaFractures: - def _report_with_meta(self, mfs): - return { - "synthesis_data": { - "situations": [ - {"title": "S1", "narrative": "a"}, - {"title": "S2", "narrative": "b"}, - ], - "thin_coverage": [], - "meta_fractures": mfs, - "metadata": {"articles_analyzed": 2}, - } - } - - def test_terminal_renders_meta_fractures(self): - report = self._report_with_meta( - [ - { - "name": "labor capacity fracture", - "description": "Whether the labor pool can absorb demand.", - "situation_indices": [0, 1], - "shared_point": "Workforce constraints.", - } - ] - ) - out = strip_ansi(BriefFormatter().format_report(report)) - assert "META-FRACTURES" in out - assert "labor capacity fracture" in out - assert "Appears in: Situation 1, Situation 2" in out - assert "Workforce constraints." in out - - def test_terminal_skips_when_empty(self): - report = self._report_with_meta([]) - out = strip_ansi(BriefFormatter().format_report(report)) - assert "META-FRACTURES" not in out - - def test_markdown_renders_meta_fractures(self): - report = self._report_with_meta( - [ - { - "name": "labor capacity fracture", - "description": "Whether the labor pool can absorb demand.", - "situation_indices": [0, 1], - "shared_point": "Workforce constraints.", - } - ] - ) - md = BriefFormatter().format_markdown(report) - assert "## Meta-fractures" in md - assert "### labor capacity fracture" in md - assert "Whether the labor pool can absorb demand." in md - assert "Workforce constraints." in md - - -class TestDecisionSummary: - def test_empty_when_no_routing(self): - assert _decision_summary({}) == [] - assert _decision_summary({"decision_routing": "not a list"}) == [] - - def test_returns_routing_list(self): - routing = [ - {"decision": "housing", "factors": [{"name": "rates", "direction": "complicates"}]} - ] - assert _decision_summary({"decision_routing": routing}) == routing - - def test_terminal_renders_decision_section(self): - report = { - "synthesis_data": { - "situations": [], - "thin_coverage": [], - "metadata": { - "articles_analyzed": 3, - "decision_routing": [ - { - "decision": "housing market monitoring", - "factors": [ - {"name": "interest rates", "direction": "complicates"}, - {"name": "inventory", "direction": "supports"}, - ], - } - ], - }, - } - } - out = strip_ansi(BriefFormatter().format_report(report)) - assert "YOUR DECISIONS" in out - assert "housing market monitoring" in out - assert "interest rates" in out - assert "complicates" in out - assert "[-]" in out # complicates glyph - assert "[+]" in out # supports glyph - - def test_markdown_renders_decision_section(self): - report = { - "synthesis_data": { - "situations": [], - "thin_coverage": [], - "metadata": { - "articles_analyzed": 3, - "decision_routing": [ - { - "decision": "career move", - "factors": [{"name": "salary range", "direction": "supports"}], - } - ], - }, - } - } - md = BriefFormatter().format_markdown(report) - assert "## Your decisions" in md - assert "### career move" in md - assert "**salary range** — supports" in md - - -class TestRenderIntegration: - """Full-report renders surface the question prefix in both terminal and md.""" - - def _report(self, primary: dict, secondary: list[dict] | None = None) -> dict: - return { - "synthesis_data": { - "situations": [ - { - "title": "Test situation", - "narrative": "Paragraph.", - "where_this_goes": { - "branching_paths": "If X then Y.", - "unresolved_questions": { - "primary": primary, - "secondary": secondary or [], - }, - "what_to_watch": "An event.", - }, - } - ], - "thin_coverage": [], - "metadata": { - "articles_analyzed": 3, - "clusters_analyzed": 1, - "clusters_thin": 0, - "analysis_threshold": "2+ articles", - }, - } - } - - def test_terminal_first_run_shows_new_tag(self): - report = self._report( - primary={ - "text": "Will A?", - "question_id": 7, - "first_asked_at": "2026-05-12T10:00:00", - "appearance_count": 1, - } - ) - out = strip_ansi(BriefFormatter().format_report(report)) - assert "Key question:" in out - assert "Q7 (new)" in out - assert "Will A?" in out - - def test_terminal_returning_question_shows_run_count(self): - report = self._report( - primary={ - "text": "Will A?", - "question_id": 7, - "first_asked_at": "2026-03-12T10:00:00", - "appearance_count": 4, - } - ) - out = strip_ansi(BriefFormatter().format_report(report)) - assert "Q7 (run 4, asked 2026-03-12)" in out - - def test_markdown_renders_question_prefix(self): - report = self._report( - primary={ - "text": "Will A?", - "question_id": 7, - "first_asked_at": "2026-03-12T10:00:00", - "appearance_count": 4, - }, - secondary=[ - { - "text": "Will B?", - "question_id": 8, - "first_asked_at": "2026-05-01T08:00:00", - "appearance_count": 1, - } - ], - ) - md = BriefFormatter().format_markdown(report) - assert "**Key question:**" in md - assert "Q7 (run 4, asked 2026-03-12)" in md - assert "Will A?" in md - assert "Q8 (new)" in md - assert "Will B?" in md - - def test_markdown_legacy_string_shape_still_renders(self): - # A synthesis emitted under the old prompt shape -- string field. - report = { - "synthesis_data": { - "situations": [ - { - "title": "Legacy", - "narrative": "X.", - "where_this_goes": { - "unresolved_question": "Old single question?", - }, - } - ], - "thin_coverage": [], - "metadata": {"articles_analyzed": 1}, - } - } - md = BriefFormatter().format_markdown(report) - assert "Old single question?" in md diff --git a/tests/cli/test_brief_from_run.py b/tests/cli/test_brief_from_run.py deleted file mode 100644 index 1aebf0c..0000000 --- a/tests/cli/test_brief_from_run.py +++ /dev/null @@ -1,183 +0,0 @@ -""" -Tests for `insightweaver brief --from-run` -- the render-only path. - -The point of these tests is what does NOT happen: no pipeline stage runs, no -API key is required, and the same stored id renders the same bytes. The -pipeline entry point is replaced with a fuse that fails the test if it is ever -awaited. -""" - -from unittest.mock import MagicMock - -import pytest - -from src.cli.brief import brief_group -from src.render.document import BriefDocument, StoredBriefNotFound -from src.render.email import EmailDeliveryError -from tests.render.conftest import SYNTHESIS_DATA -from tests.render.test_html import external_resource_hits - - -@pytest.fixture -def stored_document() -> BriefDocument: - return BriefDocument.from_synthesis_data( - SYNTHESIS_DATA, - articles_analyzed=42, - synthesis_id=176, - analysis_run_id=15, - generated_at="2026-05-15T10:43:37.873249", - ) - - -@pytest.fixture -def pipeline_fuse(monkeypatch): - """Blow up if anything reaches for the pipeline.""" - - async def _fuse(*args, **kwargs): - raise AssertionError("the render-only path must not invoke the pipeline") - - monkeypatch.setattr("src.cli.brief.run_pipeline", _fuse) - - -@pytest.fixture -def stored_brief(monkeypatch, stored_document): - monkeypatch.setattr("src.cli.brief.load_stored_brief", lambda _id: stored_document) - return stored_document - - -@pytest.fixture -def no_api_key(monkeypatch): - """ANTHROPIC_API_KEY unset, as on a machine with no Claude access.""" - settings = MagicMock() - settings.anthropic_api_key = "" - monkeypatch.setattr("src.cli.brief.settings", settings) - return settings - - -class TestRendersWithoutThePipeline: - def test_terminal_render_needs_no_api_key( - self, cli_runner, stored_brief, pipeline_fuse, no_api_key - ): - result = cli_runner.invoke(brief_group, ["--from-run", "176"]) - assert result.exit_code == 0, result.output - assert "INTELLIGENCE BRIEF" in result.output - assert "Procurement rule change lands[1]" in result.output - - def test_rendering_twice_is_byte_identical( - self, cli_runner, stored_brief, pipeline_fuse, no_api_key - ): - first = cli_runner.invoke(brief_group, ["--from-run", "176"]) - second = cli_runner.invoke(brief_group, ["--from-run", "176"]) - assert first.exit_code == second.exit_code == 0 - assert first.output == second.output - - def test_quiet_render(self, cli_runner, stored_brief, pipeline_fuse, no_api_key): - result = cli_runner.invoke(brief_group, ["--from-run", "176", "--quiet"]) - assert result.exit_code == 0, result.output - assert "Situations analyzed:" in result.output - assert "Articles: 42 | Clusters: 5" in result.output - - def test_unknown_id_reports_available_ids( - self, cli_runner, monkeypatch, pipeline_fuse, no_api_key - ): - def _missing(synthesis_id): - raise StoredBriefNotFound(synthesis_id, [189, 188]) - - monkeypatch.setattr("src.cli.brief.load_stored_brief", _missing) - result = cli_runner.invoke(brief_group, ["--from-run", "999"]) - assert result.exit_code != 0 - assert "189, 188" in result.output - - -class TestHTMLFormat: - def test_writes_a_self_contained_file( - self, cli_runner, stored_brief, pipeline_fuse, no_api_key, tmp_path - ): - target = tmp_path / "brief.html" - result = cli_runner.invoke( - brief_group, ["--from-run", "176", "--format", "html", "--output", str(target)] - ) - assert result.exit_code == 0, result.output - html = target.read_text(encoding="utf-8") - assert html.startswith("") - # tests/render/test_html.py holds the exhaustive self-containment scan; - # this checks the CLI wrote that same page rather than something else. - assert external_resource_hits(html) == [] - assert "