diff --git a/src/processors/normalizer.py b/src/processors/normalizer.py deleted file mode 100644 index 0190b84..0000000 --- a/src/processors/normalizer.py +++ /dev/null @@ -1,149 +0,0 @@ -import logging -import re -from datetime import timedelta - -from bs4 import BeautifulSoup -from sqlalchemy.orm import Session - -from src.database.models import Article -from src.utils import utcnow - -logger = logging.getLogger(__name__) - - -class ArticleNormalizer: - """Handles article content normalization and cleanup""" - - def __init__(self): - # Compile regex patterns for efficiency - self.whitespace_pattern = re.compile(r"\s+") - - def normalize_content(self, html_content: str) -> str: - """ - Convert HTML content to clean, normalized text - """ - if not html_content: - return "" - - try: - # Parse HTML - soup = BeautifulSoup(html_content, "html.parser") - - # Remove unwanted elements - for element in soup(["script", "style", "nav", "footer", "header", "aside"]): - element.decompose() - - # Extract text - text = soup.get_text() - - # Clean up whitespace - text = self.whitespace_pattern.sub(" ", text) - text = text.strip() - - return text - - except Exception as e: - logger.error(f"Error normalizing content: {e}") - return html_content - - def is_complete(self, article: Article) -> bool: - """ - Check if article has basic required fields - """ - return bool( - article.title - and article.title.strip() - and article.normalized_content - and len(article.normalized_content.strip()) > 0 - and article.published_date - ) - - def process_article(self, article: Article) -> Article: - """ - Process and normalize a single article - """ - try: - # Normalize content if not already done - if article.content and not article.normalized_content: - article.normalized_content = self.normalize_content(article.content) - article.word_count = len(article.normalized_content.split()) - - # Set language (basic detection for now) - if not article.language: - article.language = "en" # Default to English - - logger.debug(f"Normalized article {article.id}: {article.word_count} words") - - except Exception as e: - logger.error(f"Error processing article {article.id}: {e}") - - return article - - -class ArticleStorage: - """Handles article storage operations""" - - def __init__(self, db: Session): - self.db = db - self.normalizer = ArticleNormalizer() - - def store_articles(self, articles: list[Article]) -> int: - """ - Store multiple articles in the database - Returns: number of articles successfully stored - """ - stored_count = 0 - - for article in articles: - try: - # Normalize the article - article = self.normalizer.process_article(article) - - # Add to database - self.db.add(article) - stored_count += 1 - - except Exception as e: - logger.error(f"Error storing article {article.guid}: {e}") - continue - - try: - self.db.commit() - logger.info(f"Successfully stored {stored_count} articles") - except Exception as e: - logger.error(f"Error committing articles to database: {e}") - self.db.rollback() - stored_count = 0 - - return stored_count - - def get_recent_articles(self, hours: int = 48) -> list[Article]: - """ - Get articles from the last N hours - """ - cutoff_time = utcnow() - timedelta(hours=hours) - return ( - self.db.query(Article) - .filter(Article.fetched_at >= cutoff_time) - .order_by(Article.published_date.desc()) - .all() - ) - - def get_articles_by_feed(self, feed_id: int, limit: int = 100) -> list[Article]: - """ - Get recent articles from a specific feed - """ - return ( - self.db.query(Article) - .filter(Article.feed_id == feed_id) - .order_by(Article.published_date.desc()) - .limit(limit) - .all() - ) - - def get_complete_articles(self, limit: int = 100) -> list[Article]: - """ - Get articles that have all required fields - """ - articles = self.db.query(Article).order_by(Article.published_date.desc()).limit(limit).all() - return [article for article in articles if self.normalizer.is_complete(article)] diff --git a/src/rss/fetcher.py b/src/rss/fetcher.py index 772e8ea..f8b1154 100644 --- a/src/rss/fetcher.py +++ b/src/rss/fetcher.py @@ -5,16 +5,43 @@ import feedparser import httpx from bs4 import BeautifulSoup -from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session -from src.database.connection import get_db -from src.database.models import Article, RSSFeed -from src.utils import utcnow +from src.database.models import RSSFeed logger = logging.getLogger(__name__) +class LegacyWritePathClosed(RuntimeError): + """ + Raised by the legacy RSS storage entry points, which no longer write. + + Closed 2026-08-31 for backlog task 025. ``RSSFetcher.fetch_and_store_feed`` + used to insert ``Article`` rows itself, which was the one remaining way to + grow the corpus without also writing the ``Observation`` that task 014 made + the authoritative record. An article stored that way can never be routed -- + routing reads observations -- and nothing surfaces the gap: the corpus + grows and the coverage hole looks like a source problem. + + It is closed rather than rewired because the work it did is already done + correctly elsewhere: ``src/sources/rss_adapter.py`` reads the same feed + through the same normalizer and stores through ``src/sources/store.py``, + which writes the article and the observation in one transaction. Wiring + this second entry point into ``store_items`` would have preserved a + duplicate ingestion path that must be kept in step with the adapter path + forever, for no caller. Reading a feed is unaffected: ``fetch_feed`` and + ``normalize_article`` are untouched and are what the adapter uses. + """ + + +LEGACY_PATH_MESSAGE = ( + "The legacy RSS write path is closed (backlog task 025). It wrote articles " + "without observations, which Tier 1 routing cannot see. Ingest through " + "src/sources: RSSAdapter reads the feed and src.sources.store.store_items " + "writes the article and its observation in one transaction." +) + + class RSSFetcher: def __init__(self, timeout: int = 30, max_retries: int = 3): self.timeout = timeout @@ -117,126 +144,22 @@ def clean_html(self, html_content: str) -> str: return clean_text - async def fetch_and_store_feed(self, feed_id: int) -> tuple[bool, int, str | None]: + # ARG002: feed_id is unused on purpose. The parameter is kept so an existing + # caller reaches the refusal rather than a TypeError, which would read as a + # signature change rather than as a closed path (2026-08-31, task 025). + async def fetch_and_store_feed(self, feed_id: int) -> tuple[bool, int, str | None]: # noqa: ARG002 """ - Fetch a specific feed and store articles in database with enhanced retry logic - Returns: (success, articles_count, error_message) + Closed. Always raises :class:`LegacyWritePathClosed`; stores nothing. + + The signature is kept so that a caller outside this repository -- a + cron entry or a shell script -- fails loudly at the call site with a + message naming the replacement, rather than silently importing a name + that vanished or, worse, continuing to write observation-less articles. + Nothing is read and no session is opened before the raise, so a call + cannot touch the database at all: in particular it cannot stamp + ``last_fetched`` or increment ``error_count`` toward auto-deactivation. """ - with get_db() as db: - feed = db.query(RSSFeed).filter(RSSFeed.id == feed_id).first() - if not feed: - return False, 0, f"Feed with ID {feed_id} not found" - - if not feed.is_active: - return False, 0, f"Feed {feed.name} is inactive" - - # Enhanced retry logic with exponential backoff - success, feed_data, error = await self._fetch_with_retry(feed.url, feed.name) - - # Update feed status - feed.last_fetched = utcnow() - - if not success: - feed.last_error = error - feed.error_count += 1 - - # Auto-deactivate feeds with too many consecutive errors - if feed.error_count >= 10: - feed.is_active = False - logger.warning( - f"Auto-deactivated feed {feed.name} after {feed.error_count} consecutive errors" - ) - - db.commit() - return False, 0, error - - # Reset error count on success - if feed.error_count > 0: - logger.info(f"Feed {feed.name} recovered after {feed.error_count} errors") - feed.error_count = 0 - feed.last_error = None - - # Process articles - articles_count = 0 - articles_with_errors = 0 - duplicates_skipped = 0 - - for entry in feed_data.entries: - try: - article_data = self.normalize_article(entry, feed_data.feed) - - # Skip articles with insufficient data - if not article_data.get("title") or not article_data.get("guid"): - continue - - # Check if article already exists (improved deduplication check) - existing = ( - db.query(Article) - .filter(Article.feed_id == feed.id, Article.guid == article_data["guid"]) - .first() - ) - - if not existing: - article = Article(feed_id=feed.id, **article_data) - db.add(article) - articles_count += 1 - else: - duplicates_skipped += 1 - - except Exception as e: - articles_with_errors += 1 - logger.error(f"Error processing article from {feed.name}: {e}") - if articles_with_errors > 5: # Stop processing if too many article errors - logger.error( - f"Too many article processing errors for {feed.name}, stopping" - ) - break - continue - - # Commit with integrity error handling - try: - db.commit() - except IntegrityError: - db.rollback() - logger.warning( - f"Duplicate articles detected in {feed.name} during commit, skipping duplicates" - ) - # Re-process articles one by one to identify duplicates - articles_count = 0 - for entry in feed_data.entries: - try: - article_data = self.normalize_article(entry, feed_data.feed) - if not article_data.get("title") or not article_data.get("guid"): - continue - - existing = ( - db.query(Article) - .filter( - Article.feed_id == feed.id, Article.guid == article_data["guid"] - ) - .first() - ) - - if not existing: - article = Article(feed_id=feed.id, **article_data) - db.add(article) - try: - db.commit() - articles_count += 1 - except IntegrityError: - db.rollback() - duplicates_skipped += 1 - except Exception: - continue - - log_msg = f"Processed {articles_count} new articles from {feed.name}" - if duplicates_skipped > 0: - log_msg += f" ({duplicates_skipped} duplicates skipped)" - if articles_with_errors > 0: - log_msg += f" ({articles_with_errors} errors)" - - logger.info(log_msg) - return True, articles_count, None + raise LegacyWritePathClosed(LEGACY_PATH_MESSAGE) async def _fetch_with_retry( self, url: str, feed_name: str diff --git a/src/rss/parallel_fetcher.py b/src/rss/parallel_fetcher.py index 9b66378..b429b92 100644 --- a/src/rss/parallel_fetcher.py +++ b/src/rss/parallel_fetcher.py @@ -8,9 +8,8 @@ import time from dataclasses import dataclass -from src.database.connection import get_db from src.database.models import RSSFeed -from src.rss.fetcher import RSSFetcher +from src.rss.fetcher import LEGACY_PATH_MESSAGE, LegacyWritePathClosed, RSSFetcher from src.utils import utcnow logger = logging.getLogger(__name__) @@ -197,53 +196,22 @@ def _empty_results(self) -> dict[str, any]: } +# ARG001: both arguments are unused on purpose, for the same reason as +# RSSFetcher.fetch_and_store_feed -- an existing caller should hit the refusal, +# not a TypeError about an argument that used to exist (2026-08-31, task 025). async def fetch_all_active_feeds( - max_concurrent: int = 10, rate_limit: float = 2.0 + max_concurrent: int = 10, # noqa: ARG001 + rate_limit: float = 2.0, # noqa: ARG001 ) -> dict[str, any]: """ - Convenience function to fetch all active feeds from database + Closed. Always raises :class:`~src.rss.fetcher.LegacyWritePathClosed`. + + This is the function an operator reaches for to refresh the corpus, and + every article it produced arrived through ``fetch_and_store_feed``, which + no longer writes (backlog task 025, 2026-08-31). It refuses here rather + than further down for one reason: ``_rate_limited_fetch`` catches every + exception per feed and folds it into a summary dict, so a raise underneath + would return normally with "0 of 44 feeds succeeded" and a zero exit code + -- the same quiet nothing-happened this task exists to remove. """ - # Get active feed IDs and basic info from database - with get_db() as db: - active_feeds_data = ( - db.query(RSSFeed.id, RSSFeed.name, RSSFeed.url, RSSFeed.category) - .filter(RSSFeed.is_active.is_(True)) - .all() - ) - - # Sources read by a non-RSS adapter live in the same table (it is the - # sources table in everything but name) but must not be handed to - # feedparser: a JSON API endpoint would fail to parse every run and be - # auto-deactivated after ten attempts. Added 2026-08-26 for backlog task - # 005; with no such sources configured this filters nothing and behaviour - # is byte-identical to before. - from src.sources.runner import non_rss_source_urls - - adapter_urls = non_rss_source_urls() - if adapter_urls: - skipped = [f.name for f in active_feeds_data if f.url in adapter_urls] - active_feeds_data = [f for f in active_feeds_data if f.url not in adapter_urls] - if skipped: - logger.info( - f"Skipping {len(skipped)} non-RSS source(s) in the RSS fetch stage " - f"(handled by src/sources adapters): {', '.join(skipped)}" - ) - - if not active_feeds_data: - logger.warning("No active feeds found in database") - return ParallelRSSFetcher()._empty_results() - - # Convert to simple feed objects for processing - active_feeds = [] - for feed_data in active_feeds_data: - feed_obj = RSSFeed( - id=feed_data.id, name=feed_data.name, url=feed_data.url, category=feed_data.category - ) - active_feeds.append(feed_obj) - - # Create parallel fetcher and process feeds - parallel_fetcher = ParallelRSSFetcher( - max_concurrent_feeds=max_concurrent, requests_per_second=rate_limit - ) - - return await parallel_fetcher.fetch_all_feeds(active_feeds) + raise LegacyWritePathClosed(LEGACY_PATH_MESSAGE) diff --git a/src/sources/runner.py b/src/sources/runner.py index 2666dbe..0dc9472 100644 --- a/src/sources/runner.py +++ b/src/sources/runner.py @@ -220,8 +220,11 @@ def build_configured_adapters(feeds_dir: Path | str | None = None) -> list[Sourc """ Instantiate an adapter for every configured non-RSS source. - RSS feeds are excluded here because the existing ``fetch_all_active_feeds`` - path still owns them, unchanged. + RSS feeds were excluded here because ``fetch_all_active_feeds`` owned them. + That path was closed on 2026-08-31 (backlog task 025) because it wrote + articles without observations; RSS feeds are still excluded from *this* + function, which only builds the non-RSS adapters named in config, and are + now read through ``src.sources.rss_adapter.RSSAdapter`` instead. """ adapters: list[SourceAdapter] = [] for name in sorted(non_rss_adapter_names(feeds_dir)): diff --git a/tests/processors/test_normalizer.py b/tests/processors/test_normalizer.py deleted file mode 100644 index 22eb820..0000000 --- a/tests/processors/test_normalizer.py +++ /dev/null @@ -1,354 +0,0 @@ -""" -Tests for Article Normalizer -""" - -from datetime import datetime -from unittest.mock import MagicMock - -from src.processors.normalizer import ArticleNormalizer, ArticleStorage - - -class TestArticleNormalizerInit: - """Tests for ArticleNormalizer initialization""" - - def test_init_creates_pattern(self): - """Should compile whitespace pattern on init""" - normalizer = ArticleNormalizer() - - assert normalizer.whitespace_pattern is not None - - -class TestNormalizeContent: - """Tests for content normalization""" - - def test_normalize_empty_content(self): - """Should return empty string for empty input""" - normalizer = ArticleNormalizer() - - result = normalizer.normalize_content("") - - assert result == "" - - def test_normalize_none_content(self): - """Should return empty string for None input""" - normalizer = ArticleNormalizer() - - result = normalizer.normalize_content(None) - - assert result == "" - - def test_normalize_plain_text(self): - """Should handle plain text input""" - normalizer = ArticleNormalizer() - - result = normalizer.normalize_content("Plain text content") - - assert result == "Plain text content" - - def test_normalize_removes_script_tags(self): - """Should remove script elements""" - normalizer = ArticleNormalizer() - html = "

Content

" - - result = normalizer.normalize_content(html) - - assert "alert" not in result - assert "Content" in result - - def test_normalize_removes_style_tags(self): - """Should remove style elements""" - normalizer = ArticleNormalizer() - html = "

Content

" - - result = normalizer.normalize_content(html) - - assert "color" not in result - assert "Content" in result - - def test_normalize_removes_nav_tags(self): - """Should remove nav elements""" - normalizer = ArticleNormalizer() - html = "

Main content

" - - result = normalizer.normalize_content(html) - - assert "Menu" not in result - assert "Main content" in result - - def test_normalize_removes_footer_tags(self): - """Should remove footer elements""" - normalizer = ArticleNormalizer() - html = "

Content

" - - result = normalizer.normalize_content(html) - - assert "Copyright" not in result - assert "Content" in result - - def test_normalize_removes_header_tags(self): - """Should remove header elements""" - normalizer = ArticleNormalizer() - html = "
Site Header

Article content

" - - result = normalizer.normalize_content(html) - - assert "Site Header" not in result - assert "Article content" in result - - def test_normalize_removes_aside_tags(self): - """Should remove aside elements""" - normalizer = ArticleNormalizer() - html = "

Main content

" - - result = normalizer.normalize_content(html) - - assert "Sidebar" not in result - assert "Main content" in result - - def test_normalize_cleans_whitespace(self): - """Should normalize excessive whitespace""" - normalizer = ArticleNormalizer() - html = "

Content with extra spaces

" - - result = normalizer.normalize_content(html) - - assert " " not in result # No double spaces - assert "Content with extra spaces" in result - - def test_normalize_complex_html(self, sample_html_content): - """Should properly extract main content from complex HTML""" - normalizer = ArticleNormalizer() - - result = normalizer.normalize_content(sample_html_content) - - assert "Article Title" in result - assert "main article content" in result - assert "Site Header" not in result - assert "Navigation Menu" not in result - assert "Sidebar Content" not in result - assert "Site Footer" not in result - - -class TestIsComplete: - """Tests for article completeness check""" - - def test_is_complete_with_all_fields(self, sample_article_factory): - """Should return True for complete article""" - normalizer = ArticleNormalizer() - article = sample_article_factory( - title="Test Title", - normalized_content="Test content", - published_date=datetime.utcnow(), - ) - - result = normalizer.is_complete(article) - - assert result is True - - def test_is_complete_missing_title(self, sample_article_factory): - """Should return False for article without title""" - normalizer = ArticleNormalizer() - article = sample_article_factory(title="", normalized_content="Content") - - result = normalizer.is_complete(article) - - assert result is False - - def test_is_complete_missing_content(self, sample_article_factory): - """Should return False for article without normalized content""" - normalizer = ArticleNormalizer() - article = sample_article_factory(title="Title", normalized_content="") - - result = normalizer.is_complete(article) - - assert result is False - - def test_is_complete_missing_date(self, sample_article_factory): - """Should return False for article without published date""" - normalizer = ArticleNormalizer() - article = sample_article_factory( - title="Title", normalized_content="Content", published_date=None - ) - - result = normalizer.is_complete(article) - - assert result is False - - def test_is_complete_whitespace_title(self, sample_article_factory): - """Should return False for whitespace-only title""" - normalizer = ArticleNormalizer() - article = sample_article_factory(title=" ", normalized_content="Content") - - result = normalizer.is_complete(article) - - assert result is False - - -class TestProcessArticle: - """Tests for article processing""" - - def test_process_article_normalizes_content(self, sample_article_factory): - """Should normalize content if not already done""" - normalizer = ArticleNormalizer() - article = sample_article_factory(content="

HTML content

", normalized_content=None) - - result = normalizer.process_article(article) - - assert result.normalized_content == "HTML content" - - def test_process_article_sets_word_count(self, sample_article_factory): - """Should calculate word count""" - normalizer = ArticleNormalizer() - article = sample_article_factory( - content="

One two three four five

", normalized_content=None - ) - - result = normalizer.process_article(article) - - assert result.word_count == 5 - - def test_process_article_sets_default_language(self, sample_article_factory): - """Should set default language to English""" - normalizer = ArticleNormalizer() - article = sample_article_factory(content="

Test

", normalized_content=None) - article.language = None - - result = normalizer.process_article(article) - - assert result.language == "en" - - def test_process_article_skips_if_already_normalized(self, sample_article_factory): - """Should not re-normalize if content already exists""" - normalizer = ArticleNormalizer() - article = sample_article_factory( - content="

New content

", normalized_content="Existing normalized" - ) - - result = normalizer.process_article(article) - - assert result.normalized_content == "Existing normalized" - - -class TestArticleStorageInit: - """Tests for ArticleStorage initialization""" - - def test_init_with_session(self): - """Should initialize with database session""" - mock_db = MagicMock() - - storage = ArticleStorage(mock_db) - - assert storage.db == mock_db - assert storage.normalizer is not None - - -class TestStoreArticles: - """Tests for storing articles""" - - def test_store_articles_returns_count(self, sample_article_factory): - """Should return count of stored articles""" - mock_db = MagicMock() - storage = ArticleStorage(mock_db) - articles = [sample_article_factory(id=i) for i in range(3)] - - result = storage.store_articles(articles) - - assert result == 3 - assert mock_db.commit.called - - def test_store_articles_empty_list(self): - """Should handle empty article list""" - mock_db = MagicMock() - storage = ArticleStorage(mock_db) - - result = storage.store_articles([]) - - assert result == 0 - - def test_store_articles_rollback_on_commit_error(self, sample_article_factory): - """Should rollback on commit error""" - mock_db = MagicMock() - mock_db.commit.side_effect = Exception("Database error") - storage = ArticleStorage(mock_db) - articles = [sample_article_factory()] - - result = storage.store_articles(articles) - - assert result == 0 - assert mock_db.rollback.called - - -class TestGetRecentArticles: - """Tests for getting recent articles""" - - def test_get_recent_articles_default_hours(self): - """Should query articles from last 48 hours by default""" - mock_db = MagicMock() - mock_query = MagicMock() - mock_db.query.return_value = mock_query - mock_query.filter.return_value = mock_query - mock_query.order_by.return_value = mock_query - mock_query.all.return_value = [] - storage = ArticleStorage(mock_db) - - storage.get_recent_articles() - - assert mock_db.query.called - - def test_get_recent_articles_custom_hours(self): - """Should accept custom hours parameter""" - mock_db = MagicMock() - mock_query = MagicMock() - mock_db.query.return_value = mock_query - mock_query.filter.return_value = mock_query - mock_query.order_by.return_value = mock_query - mock_query.all.return_value = [] - storage = ArticleStorage(mock_db) - - storage.get_recent_articles(hours=24) - - assert mock_db.query.called - - -class TestGetArticlesByFeed: - """Tests for getting articles by feed""" - - def test_get_articles_by_feed_with_limit(self): - """Should limit results""" - mock_db = MagicMock() - mock_query = MagicMock() - mock_db.query.return_value = mock_query - mock_query.filter.return_value = mock_query - mock_query.order_by.return_value = mock_query - mock_query.limit.return_value = mock_query - mock_query.all.return_value = [] - storage = ArticleStorage(mock_db) - - storage.get_articles_by_feed(feed_id=1, limit=50) - - mock_query.limit.assert_called_with(50) - - -class TestGetCompleteArticles: - """Tests for getting complete articles""" - - def test_get_complete_articles_filters_incomplete(self, sample_article_factory): - """Should filter out incomplete articles""" - mock_db = MagicMock() - mock_query = MagicMock() - mock_db.query.return_value = mock_query - mock_query.order_by.return_value = mock_query - mock_query.limit.return_value = mock_query - - complete = sample_article_factory( - title="Complete", normalized_content="Content", published_date=datetime.utcnow() - ) - incomplete = sample_article_factory(title="", normalized_content="", published_date=None) - mock_query.all.return_value = [complete, incomplete] - - storage = ArticleStorage(mock_db) - - result = storage.get_complete_articles(limit=100) - - assert len(result) == 1 - assert result[0].title == "Complete" diff --git a/tests/rss/test_fetcher.py b/tests/rss/test_fetcher.py index a3f1c0b..b4802af 100644 --- a/tests/rss/test_fetcher.py +++ b/tests/rss/test_fetcher.py @@ -5,8 +5,15 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from sqlalchemy.orm import sessionmaker -from src.rss.fetcher import RSSFetcher, create_test_feed +from src.database.models import Article, Observation, RSSFeed +from src.rss.fetcher import ( + LEGACY_PATH_MESSAGE, + LegacyWritePathClosed, + RSSFetcher, + create_test_feed, +) class TestRSSFetcherInit: @@ -193,43 +200,131 @@ def test_clean_html_preserves_text(self): assert "Paragraph two" in result -class TestFetchAndStoreFeed: - """Tests for fetching and storing feeds""" +class TestTheLegacyWritePathIsClosed: + """ + The legacy article write path refuses instead of writing (task 025). + + ``fetch_and_store_feed`` used to insert Article rows with no Observation + beside them, which is the one way the corpus could grow rows that Tier 1 + routing can never see. It now raises. These tests drive the real entry + points -- no mock stands in for the function under test -- and check both + that the call refuses and that the database is untouched by the attempt. + """ + + @pytest.fixture + def db_pointed_at_test_engine(self, test_engine, monkeypatch): + """ + Point src.database.connection at a throwaway engine. + + The legacy path opened its own session through ``get_db`` rather than + taking one, so proving "it wrote nothing" means giving it a database it + *could* have written to and finding it empty afterwards. + """ + import src.database.connection as connection + + Session = sessionmaker(bind=test_engine) + monkeypatch.setattr(connection, "engine", test_engine) + monkeypatch.setattr(connection, "SessionLocal", Session) + session = Session() + yield session + session.close() @pytest.mark.asyncio - @patch("src.rss.fetcher.get_db") - async def test_fetch_and_store_feed_not_found(self, mock_get_db): - """Should return error for non-existent feed""" - mock_db = MagicMock() - mock_get_db.return_value.__enter__ = MagicMock(return_value=mock_db) - mock_get_db.return_value.__exit__ = MagicMock(return_value=False) - mock_db.query.return_value.filter.return_value.first.return_value = None + async def test_fetch_and_store_feed_refuses(self, db_pointed_at_test_engine): + """The call raises rather than returning a (success, count, error) tuple.""" + db = db_pointed_at_test_engine + feed = RSSFeed(name="Test Feed", url="https://example.com/feed.rss", category="news") + db.add(feed) + db.commit() fetcher = RSSFetcher() + try: + with pytest.raises(LegacyWritePathClosed, match="legacy RSS write path is closed"): + await fetcher.fetch_and_store_feed(feed.id) + finally: + await fetcher.close() - success, count, error = await fetcher.fetch_and_store_feed(999) + @pytest.mark.asyncio + async def test_the_refused_call_writes_no_article_and_no_observation( + self, db_pointed_at_test_engine + ): + """ + Articles and observations both stay at zero -- in step, at zero. + + The failure this guards against is not "it errors", it is "it errors + after inserting", which would leave exactly the unroutable rows the + task exists to prevent. + """ + db = db_pointed_at_test_engine + feed = RSSFeed(name="Test Feed", url="https://example.com/feed.rss", category="news") + db.add(feed) + db.commit() + assert db.query(Article).count() == 0 + assert db.query(Observation).count() == 0 - assert success is False - assert count == 0 - assert "not found" in error + fetcher = RSSFetcher() + try: + with pytest.raises(LegacyWritePathClosed): + await fetcher.fetch_and_store_feed(feed.id) + finally: + await fetcher.close() - await fetcher.close() + db.expire_all() + assert db.query(Article).count() == 0 + assert db.query(Observation).count() == 0 @pytest.mark.asyncio - @patch("src.rss.fetcher.get_db") - async def test_fetch_and_store_inactive_feed(self, mock_get_db, mock_inactive_feed): - """Should return error for inactive feed""" - mock_db = MagicMock() - mock_get_db.return_value.__enter__ = MagicMock(return_value=mock_db) - mock_get_db.return_value.__exit__ = MagicMock(return_value=False) - mock_db.query.return_value.filter.return_value.first.return_value = mock_inactive_feed + async def test_the_refused_call_does_not_touch_the_feed_row(self, db_pointed_at_test_engine): + """ + No last_fetched stamp and no error_count bump. + + The old path counted failures toward auto-deactivation at ten. A closed + path that still incremented that counter would quietly disable every + feed in the table after ten attempts. + """ + db = db_pointed_at_test_engine + feed = RSSFeed(name="Test Feed", url="https://example.com/feed.rss", category="news") + db.add(feed) + db.commit() fetcher = RSSFetcher() + try: + for _ in range(3): + with pytest.raises(LegacyWritePathClosed): + await fetcher.fetch_and_store_feed(feed.id) + finally: + await fetcher.close() - success, count, error = await fetcher.fetch_and_store_feed(2) + db.expire_all() + stored = db.query(RSSFeed).one() + assert stored.last_fetched is None + assert stored.error_count == 0 + assert stored.is_active is True + + def test_the_error_names_the_supported_replacement(self): + """A refusal that does not say what to use instead is a dead end.""" + assert "store_items" in LEGACY_PATH_MESSAGE + assert "RSSAdapter" in LEGACY_PATH_MESSAGE + + @pytest.mark.asyncio + async def test_reading_a_feed_still_works(self, sample_rss_response): + """ + Only the write is closed. fetch_feed and normalize_article are what + src/sources/rss_adapter.py uses, and they are untouched. + """ + fetcher = RSSFetcher() + + with patch.object(fetcher.session, "get", new_callable=AsyncMock) as mock_get: + mock_response = MagicMock() + mock_response.content = sample_rss_response + mock_response.raise_for_status = MagicMock() + mock_get.return_value = mock_response + + success, feed_data, error = await fetcher.fetch_feed("https://example.com/feed.rss") - assert success is False - assert "inactive" in error + assert success is True + assert len(feed_data.entries) == 2 + assert error is None await fetcher.close() diff --git a/tests/rss/test_parallel_fetcher.py b/tests/rss/test_parallel_fetcher.py index dd4e097..3332f2b 100644 --- a/tests/rss/test_parallel_fetcher.py +++ b/tests/rss/test_parallel_fetcher.py @@ -6,6 +6,7 @@ import pytest +from src.rss.fetcher import LegacyWritePathClosed from src.rss.parallel_fetcher import FetchResult, ParallelRSSFetcher, fetch_all_active_feeds @@ -243,42 +244,42 @@ def test_empty_results_structure(self): class TestFetchAllActiveFeeds: - """Tests for fetch_all_active_feeds convenience function""" + """ + The convenience entry point refuses too (task 025, 2026-08-31). - @pytest.mark.asyncio - @patch("src.rss.parallel_fetcher.get_db") - async def test_fetch_all_active_feeds_no_feeds(self, mock_get_db): - """Should handle no active feeds""" - mock_db = MagicMock() - mock_get_db.return_value.__enter__ = MagicMock(return_value=mock_db) - mock_get_db.return_value.__exit__ = MagicMock(return_value=False) - mock_db.query.return_value.filter.return_value.all.return_value = [] - - result = await fetch_all_active_feeds() + It is closed at the top rather than left to fail per-feed underneath: + ``_rate_limited_fetch`` catches every exception and folds it into the + summary dict, so a raise further down would return normally with zero + articles and a zero exit code -- indistinguishable from "the feeds were + quiet today", which is the silent failure this task removes. + """ - assert result["total_feeds"] == 0 + @pytest.mark.asyncio + async def test_fetch_all_active_feeds_refuses(self): + with pytest.raises(LegacyWritePathClosed, match="legacy RSS write path is closed"): + await fetch_all_active_feeds() @pytest.mark.asyncio - @patch("src.rss.parallel_fetcher.ParallelRSSFetcher") - @patch("src.rss.parallel_fetcher.get_db") - async def test_fetch_all_active_feeds_with_feeds(self, mock_get_db, mock_parallel_class): - """Should fetch active feeds from database""" - mock_db = MagicMock() - mock_get_db.return_value.__enter__ = MagicMock(return_value=mock_db) - mock_get_db.return_value.__exit__ = MagicMock(return_value=False) - - feed_data = MagicMock() - feed_data.id = 1 - feed_data.name = "Test Feed" - feed_data.url = "https://example.com/feed.rss" - feed_data.category = "news" - mock_db.query.return_value.filter.return_value.all.return_value = [feed_data] + async def test_it_refuses_before_reading_the_database(self): + """ + No session is opened, so there is nothing to mock away. - mock_fetcher = MagicMock() - mock_parallel_class.return_value = mock_fetcher - mock_fetcher.fetch_all_feeds = AsyncMock(return_value={"total_feeds": 1}) + The module no longer imports ``get_db`` at all, which is the strongest + available statement that this function cannot reach the database: the + old tests for it had to patch ``src.rss.parallel_fetcher.get_db``, and + that name is now absent. + """ + import src.rss.parallel_fetcher as module - result = await fetch_all_active_feeds(max_concurrent=5, rate_limit=1.0) + assert not hasattr(module, "get_db") + with pytest.raises(LegacyWritePathClosed): + await fetch_all_active_feeds(max_concurrent=5, rate_limit=1.0) - mock_parallel_class.assert_called_with(max_concurrent_feeds=5, requests_per_second=1.0) - assert result["total_feeds"] == 1 + @pytest.mark.asyncio + async def test_it_does_not_return_a_zero_article_summary(self): + """The failure mode being ruled out is returning normally.""" + try: + result = await fetch_all_active_feeds() + except LegacyWritePathClosed: + return + raise AssertionError(f"returned a summary instead of refusing: {result}") diff --git a/tests/sources/test_observation.py b/tests/sources/test_observation.py index a3b261b..ac17f21 100644 --- a/tests/sources/test_observation.py +++ b/tests/sources/test_observation.py @@ -197,6 +197,30 @@ def test_storing_the_same_content_again_is_a_no_op_not_an_update(self, test_sess assert test_session.query(Observation).count() == 1 +def modules_constructing(class_name: str, allowed_relative_paths: set[str]) -> list[str]: + """ + Every module under src/ that constructs ``class_name``, minus the allowed set. + + A tree grep rather than an import-graph walk, because what is being + defended against is someone *adding* a construction site, and a new file is + exactly what an import graph rooted at today's entry points would miss. + ``\\b`` before the name keeps ``NewArticle(`` from matching while still + catching the qualified ``models.Article(`` form. + """ + import re + from pathlib import Path + + root = Path(__file__).resolve().parents[2] / "src" + allowed = {root / relative for relative in allowed_relative_paths} + pattern = re.compile(rf"\b{class_name}\(") + + return [ + str(path.relative_to(root)) + for path in sorted(root.rglob("*.py")) + if path not in allowed and pattern.search(path.read_text(encoding="utf-8")) + ] + + class TestOneWritePath: def test_only_the_observation_module_constructs_an_observation(self): """ @@ -207,21 +231,31 @@ def test_only_the_observation_module_constructs_an_observation(self): rather than trusted. src/database/models.py is excluded because that is where the class is defined. """ - from pathlib import Path - - root = Path(__file__).resolve().parents[2] / "src" - allowed = {root / "sources" / "observation.py", root / "database" / "models.py"} - - offenders = [ - str(path.relative_to(root)) - for path in sorted(root.rglob("*.py")) - if path not in allowed and "Observation(" in path.read_text(encoding="utf-8") - ] + offenders = modules_constructing( + "Observation", {"sources/observation.py", "database/models.py"} + ) assert offenders == [], ( f"these modules construct an Observation directly: {offenders}. " f"Every write goes through src/sources/observation.py::store_observation." ) + def test_only_the_store_module_constructs_an_article(self): + """No module outside store.py constructs an Article. + + The grep catches construction, which is where an article write begins. A + module that only ``db.add()``s an Article it was handed would slip past it -- + ``src/processors/normalizer.py::ArticleStorage`` was exactly that, dead code + kept alive by its own tests, and it was deleted in task 025 rather than + documented, because a documented hole in an invariant is still a hole. + """ + offenders = modules_constructing("Article", {"sources/store.py", "database/models.py"}) + assert offenders == [], ( + f"these modules construct an Article directly: {offenders}. " + f"Every article write goes through src/sources/store.py::store_items, " + f"which writes the article and its Observation in one transaction. " + f"See src/rss/fetcher.py::LegacyWritePathClosed for why." + ) + def test_the_adapter_store_path_writes_an_observation_per_article(self, test_session, source): items = [make_item(), make_item(guid="doc-2", title="A different rule", url="u2")] inserted, duplicates = store_items(test_session, source, items)