From 9f1fb66a4962df16a6bb8c92c4aa00749a0f8f6d Mon Sep 17 00:00:00 2001 From: audemed44 Date: Sat, 28 Mar 2026 17:20:10 +0530 Subject: [PATCH] feat(serials): preserve stubbed chapters --- ...8a9b21_preserve_stubbed_serial_chapters.py | 96 ++++++ backend/app/models/serial.py | 14 + backend/app/routers/serials.py | 57 ++-- backend/app/schemas/serial.py | 13 + backend/app/services/serial_service.py | 288 ++++++++++++------ backend/tests/test_serial_service.py | 188 +++++++++++- frontend/src/__tests__/ChapterList.test.tsx | 2 + frontend/src/__tests__/SerialDetail.test.tsx | 2 + frontend/src/__tests__/Serials.test.tsx | 6 + frontend/src/__tests__/VolumeList.test.tsx | 2 + .../src/components/serials/ChapterList.tsx | 5 + .../src/components/serials/SerialCard.tsx | 8 +- .../src/components/serials/VolumeList.tsx | 21 +- frontend/src/pages/Home.tsx | 5 + frontend/src/pages/SerialDetail.tsx | 8 + frontend/src/types/api.ts | 11 + 16 files changed, 605 insertions(+), 121 deletions(-) create mode 100644 backend/alembic/versions/3c1d7f8a9b21_preserve_stubbed_serial_chapters.py diff --git a/backend/alembic/versions/3c1d7f8a9b21_preserve_stubbed_serial_chapters.py b/backend/alembic/versions/3c1d7f8a9b21_preserve_stubbed_serial_chapters.py new file mode 100644 index 0000000..9c5ad17 --- /dev/null +++ b/backend/alembic/versions/3c1d7f8a9b21_preserve_stubbed_serial_chapters.py @@ -0,0 +1,96 @@ +"""preserve stubbed serial chapters + +Revision ID: 3c1d7f8a9b21 +Revises: 6305b1bb905b +Create Date: 2026-03-28 17:35:00.000000 + +""" + +from collections.abc import Sequence +from urllib.parse import urlparse + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "3c1d7f8a9b21" +down_revision: str | None = "6305b1bb905b" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _normalize_source_key(url: str) -> str: + parsed = urlparse(url) + return parsed._replace(fragment="").geturl() + + +def upgrade() -> None: + with op.batch_alter_table("web_serials") as batch_op: + batch_op.add_column( + sa.Column( + "live_chapter_count", + sa.Integer(), + nullable=False, + server_default="0", + ) + ) + + with op.batch_alter_table("serial_chapters") as batch_op: + batch_op.add_column(sa.Column("source_key", sa.Text(), nullable=True)) + batch_op.add_column( + sa.Column( + "is_stubbed", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ) + ) + batch_op.add_column(sa.Column("stubbed_at", sa.DateTime(), nullable=True)) + + bind = op.get_bind() + chapters = bind.execute(sa.text("SELECT id, source_url FROM serial_chapters")).fetchall() + for chapter_id, source_url in chapters: + bind.execute( + sa.text("UPDATE serial_chapters SET source_key = :source_key WHERE id = :chapter_id"), + { + "chapter_id": chapter_id, + "source_key": _normalize_source_key(source_url), + }, + ) + op.execute( + """ + UPDATE web_serials + SET total_chapters = COALESCE( + (SELECT MAX(ch.chapter_number) + FROM serial_chapters AS ch + WHERE ch.serial_id = web_serials.id), + 0 + ), + live_chapter_count = COALESCE( + (SELECT MAX(ch.chapter_number) + FROM serial_chapters AS ch + WHERE ch.serial_id = web_serials.id), + 0 + ) + """ + ) + + with op.batch_alter_table("serial_chapters") as batch_op: + batch_op.alter_column("source_key", existing_type=sa.Text(), nullable=False) + batch_op.create_index( + "ix_serial_chapters_serial_id_source_key", + ["serial_id", "source_key"], + unique=True, + ) + + +def downgrade() -> None: + with op.batch_alter_table("serial_chapters") as batch_op: + batch_op.drop_index("ix_serial_chapters_serial_id_source_key") + batch_op.drop_column("stubbed_at") + batch_op.drop_column("is_stubbed") + batch_op.drop_column("source_key") + + with op.batch_alter_table("web_serials") as batch_op: + batch_op.drop_column("live_chapter_count") diff --git a/backend/app/models/serial.py b/backend/app/models/serial.py index 0730c12..33634e7 100644 --- a/backend/app/models/serial.py +++ b/backend/app/models/serial.py @@ -23,6 +23,7 @@ class WebSerial(Base): Text, default="ongoing", nullable=False ) # ongoing|completed|paused|error total_chapters: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + live_chapter_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) last_checked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) last_error: Mapped[str | None] = mapped_column(Text, nullable=True) source_metadata: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON blob @@ -42,6 +43,10 @@ class WebSerial(Base): ) series: Mapped["Series | None"] = relationship("Series") # type: ignore[name-defined] # noqa: F821 + @property + def stubbed_chapter_count(self) -> int: + return max(self.total_chapters - self.live_chapter_count, 0) + class SerialChapter(Base): __tablename__ = "serial_chapters" @@ -52,6 +57,12 @@ class SerialChapter(Base): "chapter_number", unique=True, ), + Index( + "ix_serial_chapters_serial_id_source_key", + "serial_id", + "source_key", + unique=True, + ), ) id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) @@ -59,6 +70,7 @@ class SerialChapter(Base): Integer, ForeignKey("web_serials.id", ondelete="CASCADE"), nullable=False ) chapter_number: Mapped[int] = mapped_column(Integer, nullable=False) # 1-based + source_key: Mapped[str] = mapped_column(Text, nullable=False) title: Mapped[str | None] = mapped_column(Text, nullable=True) source_url: Mapped[str] = mapped_column(Text, nullable=False) publish_date: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) @@ -67,6 +79,8 @@ class SerialChapter(Base): ) # cleaned HTML, null until fetched word_count: Mapped[int | None] = mapped_column(Integer, nullable=True) fetched_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + is_stubbed: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + stubbed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) serial: Mapped["WebSerial"] = relationship("WebSerial", back_populates="chapters") diff --git a/backend/app/routers/serials.py b/backend/app/routers/serials.py index 2170efd..9359400 100644 --- a/backend/app/routers/serials.py +++ b/backend/app/routers/serials.py @@ -43,7 +43,7 @@ generate_volume, get_chapter_fetch_status, get_serial, - get_volume_word_counts, + get_volume_metrics, list_chapter_responses, list_serials, list_serials_for_dashboard, @@ -66,15 +66,22 @@ WORDS_PER_PAGE = 280 -def _enrich_volumes(volumes: list[object], word_counts: dict[int, int]) -> list[VolumeResponse]: - """Build VolumeResponse list with estimated_pages and total_words.""" +def _enrich_volumes(volumes: list[object], metrics: dict[int, object]) -> list[VolumeResponse]: + """Build VolumeResponse list with derived volume metrics.""" result: list[VolumeResponse] = [] for v in volumes: resp = VolumeResponse.model_validate(v) - words = word_counts.get(resp.id) - if words is not None and words > 0: - resp.total_words = words - resp.estimated_pages = max(1, words // WORDS_PER_PAGE) + metric = metrics.get(resp.id) + if metric is not None: + total_words = int(getattr(metric, "total_words")) + resp.total_words = total_words + resp.estimated_pages = ( + max(1, total_words // WORDS_PER_PAGE) if total_words > 0 else None + ) + resp.chapter_count = int(getattr(metric, "chapter_count")) + resp.fetched_chapter_count = int(getattr(metric, "fetched_chapter_count")) + resp.is_partial = bool(getattr(metric, "is_partial")) + resp.stubbed_missing_count = int(getattr(metric, "stubbed_missing_count")) result.append(resp) return result @@ -342,8 +349,8 @@ async def configure_volumes_endpoint( except Exception as exc: log.exception("Failed to configure volumes for serial %d", serial_id) raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) - word_counts = await get_volume_word_counts(session, serial_id) - return _enrich_volumes(volumes, word_counts) + metrics = await get_volume_metrics(session, serial_id) + return _enrich_volumes(volumes, metrics) @router.post( @@ -356,8 +363,8 @@ async def auto_split_volumes_endpoint( volumes = await auto_split_volumes(session, serial_id, body) except SerialNotFound as exc: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) - word_counts = await get_volume_word_counts(session, serial_id) - return _enrich_volumes(volumes, word_counts) + metrics = await get_volume_metrics(session, serial_id) + return _enrich_volumes(volumes, metrics) @router.post("/serials/{serial_id}/volumes/add", response_model=VolumeResponse, status_code=201) @@ -368,8 +375,8 @@ async def add_single_volume_endpoint( vol = await add_single_volume(session, serial_id, body) except SerialNotFound as exc: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) - word_counts = await get_volume_word_counts(session, serial_id) - return _enrich_volumes([vol], word_counts)[0] + metrics = await get_volume_metrics(session, serial_id) + return _enrich_volumes([vol], metrics)[0] @router.get("/serials/{serial_id}/volumes", response_model=list[VolumeResponse]) @@ -378,8 +385,8 @@ async def list_volumes_endpoint(serial_id: int, session: AsyncSession = Depends( volumes = await list_volumes(session, serial_id) except SerialNotFound as exc: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) - word_counts = await get_volume_word_counts(session, serial_id) - return _enrich_volumes(volumes, word_counts) + metrics = await get_volume_metrics(session, serial_id) + return _enrich_volumes(volumes, metrics) @router.patch("/serials/{serial_id}/volumes/{volume_id}", response_model=VolumeResponse) @@ -393,8 +400,8 @@ async def update_volume_endpoint( vol = await update_volume(session, serial_id, volume_id, body) except SerialNotFound as exc: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) - word_counts = await get_volume_word_counts(session, serial_id) - return _enrich_volumes([vol], word_counts)[0] + metrics = await get_volume_metrics(session, serial_id) + return _enrich_volumes([vol], metrics)[0] @router.delete("/serials/{serial_id}/volumes/{volume_id}", status_code=status.HTTP_204_NO_CONTENT) @@ -426,8 +433,8 @@ async def upload_volume_cover_endpoint( vol = await upload_volume_cover(session, serial_id, volume_id, content, suffix) except SerialNotFound as exc: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) - word_counts = await get_volume_word_counts(session, serial_id) - return _enrich_volumes([vol], word_counts)[0] + metrics = await get_volume_metrics(session, serial_id) + return _enrich_volumes([vol], metrics)[0] @router.post("/serials/{serial_id}/volumes/{volume_id}/generate", response_model=VolumeResponse) @@ -447,8 +454,8 @@ async def generate_volume_endpoint( except Exception as exc: log.exception("Unexpected error generating volume %d for serial %d", volume_id, serial_id) raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) - word_counts = await get_volume_word_counts(session, serial_id) - return _enrich_volumes([vol], word_counts)[0] + metrics = await get_volume_metrics(session, serial_id) + return _enrich_volumes([vol], metrics)[0] @router.post("/serials/{serial_id}/volumes/generate-all", response_model=list[VolumeResponse]) @@ -461,8 +468,8 @@ async def generate_all_volumes_endpoint( volumes = await generate_all_volumes(session, serial_id, shelf_id) except SerialNotFound as exc: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) - word_counts = await get_volume_word_counts(session, serial_id) - return _enrich_volumes(volumes, word_counts) + metrics = await get_volume_metrics(session, serial_id) + return _enrich_volumes(volumes, metrics) @router.post("/serials/{serial_id}/volumes/{volume_id}/rebuild", response_model=VolumeResponse) @@ -477,5 +484,5 @@ async def rebuild_volume_endpoint( raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) except VolumeGenerationError as exc: raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) - word_counts = await get_volume_word_counts(session, serial_id) - return _enrich_volumes([vol], word_counts)[0] + metrics = await get_volume_metrics(session, serial_id) + return _enrich_volumes([vol], metrics)[0] diff --git a/backend/app/schemas/serial.py b/backend/app/schemas/serial.py index aecb967..9dce8ce 100644 --- a/backend/app/schemas/serial.py +++ b/backend/app/schemas/serial.py @@ -30,6 +30,8 @@ class SerialResponse(BaseModel): cover_url: str | None status: str total_chapters: int + live_chapter_count: int + stubbed_chapter_count: int last_checked_at: datetime | None last_error: str | None created_at: datetime @@ -43,6 +45,8 @@ class SerialDashboardResponse(BaseModel): cover_path: str | None status: str total_chapters: int + live_chapter_count: int + stubbed_chapter_count: int fetched_count: int new_chapter_count: int latest_chapter_title: str | None @@ -57,6 +61,8 @@ class ChapterResponse(BaseModel): chapter_number: int title: str | None source_url: str + is_stubbed: bool = False + stubbed_at: datetime | None = None publish_date: datetime | None word_count: int | None estimated_pages: int | None = None @@ -85,6 +91,8 @@ def from_orm( chapter_number=obj.chapter_number, title=obj.title, source_url=obj.source_url, + is_stubbed=obj.is_stubbed, + stubbed_at=obj.stubbed_at, publish_date=obj.publish_date, word_count=obj.word_count, estimated_pages=estimated_pages, @@ -171,6 +179,10 @@ class VolumeResponse(BaseModel): chapter_end: int generated_at: datetime | None is_stale: bool + chapter_count: int = 0 + fetched_chapter_count: int = 0 + is_partial: bool = False + stubbed_missing_count: int = 0 estimated_pages: int | None = None total_words: int | None = None @@ -184,3 +196,4 @@ class VolumePreviewResponse(BaseModel): total_words: int = 0 estimated_pages: int | None = None is_partial: bool = False + stubbed_missing_count: int = 0 diff --git a/backend/app/services/serial_service.py b/backend/app/services/serial_service.py index f11627a..9f517e0 100644 --- a/backend/app/services/serial_service.py +++ b/backend/app/services/serial_service.py @@ -30,6 +30,7 @@ VolumeRange, VolumeUpdate, ) +from app.scrapers.base import ChapterInfo, normalize_url from app.scrapers.registry import get_adapter, get_adapter_by_name log = logging.getLogger(__name__) @@ -244,6 +245,103 @@ def _covers_dir() -> Path: return Path(get_settings().covers_dir) +@dataclass(frozen=True) +class VolumeMetrics: + total_words: int + fetched_chapter_count: int + chapter_count: int + is_partial: bool + stubbed_missing_count: int + + +def _chapter_source_key(source_url: str) -> str: + return normalize_url(source_url) + + +async def _mark_generated_volumes_stale( + session: AsyncSession, serial_id: int, changed_numbers: set[int] +) -> None: + if not changed_numbers: + return + + vol_result = await session.execute( + select(SerialVolume).where( + SerialVolume.serial_id == serial_id, + SerialVolume.generated_at.isnot(None), + ) + ) + for vol in vol_result.scalars().all(): + if any(vol.chapter_start <= n <= vol.chapter_end for n in changed_numbers): + vol.is_stale = True + + +async def _sync_remote_chapter_list( + session: AsyncSession, + serial: WebSerial, + remote_chapters: list[ChapterInfo], +) -> tuple[int, set[int]]: + existing_result = await session.execute( + select(SerialChapter) + .where(SerialChapter.serial_id == serial.id) + .order_by(SerialChapter.chapter_number) + ) + existing_chapters = list(existing_result.scalars().all()) + existing_by_key = {chapter.source_key: chapter for chapter in existing_chapters} + next_number = max((chapter.chapter_number for chapter in existing_chapters), default=0) + 1 + + now = datetime.now(UTC) + seen_keys: set[str] = set() + changed_numbers: set[int] = set() + new_count = 0 + + for remote in remote_chapters: + source_key = _chapter_source_key(remote.source_url) + if source_key in seen_keys: + continue + seen_keys.add(source_key) + + existing = existing_by_key.get(source_key) + if existing is None: + session.add( + SerialChapter( + serial_id=serial.id, + chapter_number=next_number, + source_key=source_key, + title=remote.title, + source_url=remote.source_url, + publish_date=remote.publish_date, + is_stubbed=False, + ) + ) + changed_numbers.add(next_number) + next_number += 1 + new_count += 1 + continue + + existing.title = remote.title + existing.source_url = remote.source_url + existing.publish_date = remote.publish_date + if existing.is_stubbed: + existing.is_stubbed = False + existing.stubbed_at = None + changed_numbers.add(existing.chapter_number) + + live_chapter_count = 0 + for chapter in existing_chapters: + if chapter.source_key in seen_keys: + live_chapter_count += 1 + continue + if not chapter.is_stubbed: + chapter.is_stubbed = True + chapter.stubbed_at = now + changed_numbers.add(chapter.chapter_number) + + serial.total_chapters = max(next_number - 1, 0) + serial.live_chapter_count = live_chapter_count + new_count + + return new_count, changed_numbers + + # --------------------------------------------------------------------------- # Serial CRUD # --------------------------------------------------------------------------- @@ -295,23 +393,15 @@ async def add_serial( cover_path=cover_path, cover_url=metadata.cover_url, status=metadata.status, - total_chapters=len(chapters), + total_chapters=0, + live_chapter_count=0, last_checked_at=datetime.now(UTC), series_id=series.id, ) session.add(serial) await session.flush() # get serial.id - for ch in chapters: - session.add( - SerialChapter( - serial_id=serial.id, - chapter_number=ch.chapter_number, - title=ch.title, - source_url=ch.source_url, - publish_date=ch.publish_date, - ) - ) + await _sync_remote_chapter_list(session, serial, chapters) await session.commit() await session.refresh(serial) @@ -542,6 +632,12 @@ async def fetch_chapters_content( if progress is not None: await progress.mark_chapter_skipped(chapter) continue + if chapter.is_stubbed: + message = "chapter is stubbed upstream and has no cached content" + log.warning("Skipping stubbed chapter %d: %s", chapter.chapter_number, message) + if progress is not None: + await progress.mark_chapter_failed(chapter, message) + continue try: log.info("Fetching chapter %d: %s", chapter.chapter_number, chapter.source_url) if progress is not None: @@ -566,16 +662,8 @@ async def fetch_chapters_content( if progress is not None: await progress.mark_chapter_failed(chapter, str(exc)) - # Mark volumes covering fetched chapters as stale if newly_fetched_numbers: - vol_result = await session.execute( - select(SerialVolume).where(SerialVolume.serial_id == serial_id) - ) - for vol in vol_result.scalars().all(): - if any(vol.chapter_start <= n <= vol.chapter_end for n in newly_fetched_numbers): - if vol.generated_at is not None: - vol.is_stale = True - + await _mark_generated_volumes_stale(session, serial_id, newly_fetched_numbers) await session.commit() return fetched @@ -599,45 +687,16 @@ async def update_from_source(session: AsyncSession, serial_id: int) -> dict: await session.commit() raise ScrapingError(str(exc)) from exc - # Find existing chapter numbers - existing_result = await session.execute( - select(SerialChapter.chapter_number).where(SerialChapter.serial_id == serial_id) - ) - existing_numbers = {row[0] for row in existing_result.all()} - - new_chapters = [ch for ch in chapters if ch.chapter_number not in existing_numbers] - for ch in new_chapters: - session.add( - SerialChapter( - serial_id=serial_id, - chapter_number=ch.chapter_number, - title=ch.title, - source_url=ch.source_url, - publish_date=ch.publish_date, - ) - ) - - serial.total_chapters = len(chapters) + new_count, changed_numbers = await _sync_remote_chapter_list(session, serial, chapters) serial.last_checked_at = datetime.now(UTC) serial.last_error = None if serial.status == "error": serial.status = "ongoing" - # Mark volumes as stale if new chapters fall within their range - if new_chapters: - new_numbers = {ch.chapter_number for ch in new_chapters} - vol_result = await session.execute( - select(SerialVolume).where( - SerialVolume.serial_id == serial_id, - SerialVolume.generated_at.isnot(None), - ) - ) - for vol in vol_result.scalars().all(): - if any(vol.chapter_start <= n <= vol.chapter_end for n in new_numbers): - vol.is_stale = True + await _mark_generated_volumes_stale(session, serial_id, changed_numbers) await session.commit() - return {"new_chapters": len(new_chapters), "total_chapters": len(chapters)} + return {"new_chapters": new_count, "total_chapters": serial.total_chapters} # --------------------------------------------------------------------------- @@ -995,23 +1054,68 @@ async def add_single_volume( # --------------------------------------------------------------------------- -async def get_volume_word_counts(session: AsyncSession, serial_id: int) -> dict[int, int]: - """Return {volume_id: total_words} for all volumes of the serial.""" +async def get_volume_metrics(session: AsyncSession, serial_id: int) -> dict[int, VolumeMetrics]: + """Return derived metrics for all configured volumes of a serial.""" + volumes = await list_volumes(session, serial_id) + if not volumes: + return {} + + min_chapter = min(volume.chapter_start for volume in volumes) + max_chapter = max(volume.chapter_end for volume in volumes) result = await session.execute( select( - SerialVolume.id, - sa_func.coalesce(sa_func.sum(SerialChapter.word_count), 0), + SerialChapter.chapter_number, + SerialChapter.word_count, + SerialChapter.content, + SerialChapter.is_stubbed, ) - .join( - SerialChapter, - (SerialChapter.serial_id == SerialVolume.serial_id) - & (SerialChapter.chapter_number >= SerialVolume.chapter_start) - & (SerialChapter.chapter_number <= SerialVolume.chapter_end), + .where( + SerialChapter.serial_id == serial_id, + SerialChapter.chapter_number >= min_chapter, + SerialChapter.chapter_number <= max_chapter, ) - .where(SerialVolume.serial_id == serial_id) - .group_by(SerialVolume.id) + .order_by(SerialChapter.chapter_number) ) - return {row[0]: int(row[1]) for row in result.all()} + chapter_data = { + int(row[0]): { + "word_count": row[1], + "has_content": row[2] is not None, + "is_stubbed": bool(row[3]), + } + for row in result.all() + } + + metrics: dict[int, VolumeMetrics] = {} + for volume in volumes: + total_words = 0 + fetched_chapter_count = 0 + stubbed_missing_count = 0 + is_partial = False + + for chapter_number in range(volume.chapter_start, volume.chapter_end + 1): + data = chapter_data.get(chapter_number) + if data is None or data["word_count"] is None: + is_partial = True + if data is not None and data["is_stubbed"] and not data["has_content"]: + stubbed_missing_count += 1 + continue + fetched_chapter_count += 1 + total_words += int(data["word_count"]) + + metrics[volume.id] = VolumeMetrics( + total_words=total_words, + fetched_chapter_count=fetched_chapter_count, + chapter_count=max(0, volume.chapter_end - volume.chapter_start + 1), + is_partial=is_partial, + stubbed_missing_count=stubbed_missing_count, + ) + + return metrics + + +async def get_volume_word_counts(session: AsyncSession, serial_id: int) -> dict[int, int]: + metrics = await get_volume_metrics(session, serial_id) + return {volume_id: metric.total_words for volume_id, metric in metrics.items()} async def preview_volume_ranges( @@ -1027,7 +1131,12 @@ async def preview_volume_ranges( min_chapter = min(split.start for split in splits) max_chapter = max(split.end for split in splits) result = await session.execute( - select(SerialChapter.chapter_number, SerialChapter.word_count) + select( + SerialChapter.chapter_number, + SerialChapter.word_count, + SerialChapter.content, + SerialChapter.is_stubbed, + ) .where( SerialChapter.serial_id == serial_id, SerialChapter.chapter_number >= min_chapter, @@ -1035,7 +1144,7 @@ async def preview_volume_ranges( ) .order_by(SerialChapter.chapter_number) ) - chapter_word_counts = {int(row[0]): row[1] for row in result.all()} + chapter_word_counts = {int(row[0]): (row[1], row[2], bool(row[3])) for row in result.all()} previews: list[VolumePreviewResponse] = [] for split in splits: @@ -1043,11 +1152,15 @@ async def preview_volume_ranges( total_words = 0 fetched_chapter_count = 0 is_partial = False + stubbed_missing_count = 0 for chapter_number in chapter_numbers: - word_count = chapter_word_counts.get(chapter_number) + row = chapter_word_counts.get(chapter_number) + word_count = row[0] if row is not None else None if word_count is None: is_partial = True + if row is not None and row[2] and row[1] is None: + stubbed_missing_count += 1 continue fetched_chapter_count += 1 total_words += int(word_count) @@ -1063,6 +1176,7 @@ async def preview_volume_ranges( total_words=total_words, estimated_pages=_estimate_pages(total_words), is_partial=is_partial, + stubbed_missing_count=stubbed_missing_count, ) ) @@ -1123,31 +1237,13 @@ async def check_serial_for_updates(session: AsyncSession, serial: WebSerial) -> log.warning("Failed to fetch chapter list for serial %d: %s", serial.id, exc) return 0 - # Get existing chapter numbers - result = await session.execute( - select(SerialChapter.chapter_number).where(SerialChapter.serial_id == serial.id) - ) - existing_numbers = set(result.scalars().all()) - - new_count = 0 - for ch in remote_chapters: - if ch.chapter_number not in existing_numbers: - session.add( - SerialChapter( - serial_id=serial.id, - chapter_number=ch.chapter_number, - title=ch.title, - source_url=ch.source_url, - publish_date=ch.publish_date, - ) - ) - new_count += 1 - - serial.total_chapters = len(remote_chapters) + new_count, changed_numbers = await _sync_remote_chapter_list(session, serial, remote_chapters) serial.last_checked_at = datetime.now(UTC) + serial.last_error = None if serial.status == "error": serial.status = "ongoing" - serial.last_error = None + + await _mark_generated_volumes_stale(session, serial.id, changed_numbers) await session.commit() if new_count > 0: @@ -1194,6 +1290,8 @@ class SerialDashboardEntry: cover_path: str | None status: str total_chapters: int + live_chapter_count: int + stubbed_chapter_count: int fetched_count: int new_chapter_count: int latest_chapter_title: str | None @@ -1218,7 +1316,10 @@ async def list_serials_for_dashboard( count_q = ( select(sa_func.count()) .select_from(SerialChapter) - .where(SerialChapter.serial_id == serial.id) + .where( + SerialChapter.serial_id == serial.id, + SerialChapter.is_stubbed.is_(False), + ) ) if serial.last_viewed_at is not None: count_q = count_q.where(SerialChapter.publish_date > serial.last_viewed_at) @@ -1240,7 +1341,10 @@ async def list_serials_for_dashboard( # Get latest chapter latest = await session.scalar( select(SerialChapter) - .where(SerialChapter.serial_id == serial.id) + .where( + SerialChapter.serial_id == serial.id, + SerialChapter.is_stubbed.is_(False), + ) .order_by(SerialChapter.chapter_number.desc()) .limit(1) ) @@ -1253,6 +1357,8 @@ async def list_serials_for_dashboard( cover_path=serial.cover_path, status=serial.status, total_chapters=serial.total_chapters, + live_chapter_count=serial.live_chapter_count, + stubbed_chapter_count=serial.stubbed_chapter_count, fetched_count=fetched, new_chapter_count=new_count, latest_chapter_title=latest.title if latest else None, diff --git a/backend/tests/test_serial_service.py b/backend/tests/test_serial_service.py index 078c324..4ea1074 100644 --- a/backend/tests/test_serial_service.py +++ b/backend/tests/test_serial_service.py @@ -82,6 +82,7 @@ async def serial(db_session): author="Author", status="ongoing", total_chapters=3, + live_chapter_count=3, series_id=series.id, ) db_session.add(s) @@ -92,8 +93,9 @@ async def serial(db_session): SerialChapter( serial_id=s.id, chapter_number=i, + source_key=f"https://royalroad.com/ch/{i}", title=f"Chapter {i}", - source_url=f"https://royalroad.com/fiction/1/chapter/{i}", + source_url=f"https://royalroad.com/ch/{i}", ) ) @@ -757,6 +759,35 @@ async def test_fetch_chapters_error_sets_status(db_session, serial): assert serial.last_error is not None +@pytest.mark.asyncio +async def test_fetch_stubbed_chapter_without_cache_skips_scraper(db_session, serial): + from sqlalchemy import select + + result = await db_session.execute( + select(SerialChapter).where( + SerialChapter.serial_id == serial.id, + SerialChapter.chapter_number == 2, + ) + ) + chapter = result.scalar_one() + chapter.is_stubbed = True + chapter.stubbed_at = datetime.now(UTC) + chapter.content = None + await db_session.commit() + + mock_adapter = MagicMock() + mock_adapter.fetch_chapter_content = AsyncMock() + + with patch("app.services.serial_service.get_adapter_by_name", return_value=mock_adapter): + fetched = await fetch_chapters_content(db_session, serial.id, 2, 2) + + assert fetched == [] + mock_adapter.fetch_chapter_content.assert_not_called() + await db_session.refresh(serial) + assert serial.status == "ongoing" + assert serial.last_error is None + + @pytest.mark.asyncio async def test_update_from_source_no_adapter(db_session, serial): with ( @@ -817,6 +848,116 @@ async def test_update_from_source_marks_volumes_stale(db_session, serial): assert vol.is_stale is True +@pytest.mark.asyncio +async def test_update_from_source_preserves_stubbed_chapter_numbers(db_session, serial): + from sqlalchemy import select + + from app.scrapers.base import ChapterInfo + + for i in range(4, 101): + db_session.add( + SerialChapter( + serial_id=serial.id, + chapter_number=i, + source_key=f"https://royalroad.com/ch/{i}", + title=f"Ch {i}", + source_url=f"https://royalroad.com/ch/{i}", + ) + ) + serial.total_chapters = 100 + serial.live_chapter_count = 100 + await db_session.commit() + + surviving_numbers = [1, 2, 3, 4, *range(51, 101)] + mock_adapter = MagicMock() + mock_adapter.fetch_chapter_list = AsyncMock( + return_value=[ + ChapterInfo(idx, f"Ch {number}", f"https://royalroad.com/ch/{number}", None) + for idx, number in enumerate(surviving_numbers, start=1) + ] + ) + + with patch("app.services.serial_service.get_adapter_by_name", return_value=mock_adapter): + result = await update_from_source(db_session, serial.id) + + assert result["new_chapters"] == 0 + assert result["total_chapters"] == 100 + + await db_session.refresh(serial) + assert serial.total_chapters == 100 + assert serial.live_chapter_count == 54 + assert serial.stubbed_chapter_count == 46 + + chapters = list( + ( + await db_session.execute( + select(SerialChapter) + .where(SerialChapter.serial_id == serial.id) + .order_by(SerialChapter.chapter_number) + ) + ).scalars() + ) + assert chapters[4].chapter_number == 5 + assert chapters[4].is_stubbed is True + assert chapters[50].chapter_number == 51 + assert chapters[50].is_stubbed is False + + +@pytest.mark.asyncio +async def test_update_from_source_appends_new_chapters_after_stubbed_gap(db_session, serial): + from sqlalchemy import select + + from app.scrapers.base import ChapterInfo + + for i in range(4, 101): + db_session.add( + SerialChapter( + serial_id=serial.id, + chapter_number=i, + source_key=f"https://royalroad.com/ch/{i}", + title=f"Ch {i}", + source_url=f"https://royalroad.com/ch/{i}", + ) + ) + serial.total_chapters = 100 + serial.live_chapter_count = 100 + await db_session.commit() + + surviving_and_new = [1, 2, 3, 4, *range(51, 103)] + mock_adapter = MagicMock() + mock_adapter.fetch_chapter_list = AsyncMock( + return_value=[ + ChapterInfo(idx, f"Ch {number}", f"https://royalroad.com/ch/{number}", None) + for idx, number in enumerate(surviving_and_new, start=1) + ] + ) + + with patch("app.services.serial_service.get_adapter_by_name", return_value=mock_adapter): + result = await update_from_source(db_session, serial.id) + + assert result["new_chapters"] == 2 + assert result["total_chapters"] == 102 + + await db_session.refresh(serial) + assert serial.total_chapters == 102 + assert serial.live_chapter_count == 56 + assert serial.stubbed_chapter_count == 46 + + chapters = list( + ( + await db_session.execute( + select(SerialChapter) + .where(SerialChapter.serial_id == serial.id) + .order_by(SerialChapter.chapter_number) + ) + ).scalars() + ) + assert chapters[-2].chapter_number == 101 + assert chapters[-2].source_url == "https://royalroad.com/ch/101" + assert chapters[-1].chapter_number == 102 + assert chapters[-1].source_url == "https://royalroad.com/ch/102" + + @pytest.mark.asyncio async def test_configure_volumes_replaces_ungenerated(db_session, serial): """configure_volumes deletes existing volumes that have no book_id.""" @@ -1583,6 +1724,7 @@ async def test_api_volume_preview_reports_partial_estimates(client, db_session, "total_words": 280, "estimated_pages": 1, "is_partial": True, + "stubbed_missing_count": 0, }, { "start": 2, @@ -1593,10 +1735,50 @@ async def test_api_volume_preview_reports_partial_estimates(client, db_session, "total_words": 560, "estimated_pages": 2, "is_partial": True, + "stubbed_missing_count": 0, }, ] +@pytest.mark.asyncio +async def test_api_volume_preview_counts_stubbed_missing_chapters(client, db_session, serial): + from sqlalchemy import select + + result = await db_session.execute( + select(SerialChapter) + .where(SerialChapter.serial_id == serial.id) + .order_by(SerialChapter.chapter_number) + ) + chapters = list(result.scalars().all()) + chapters[0].word_count = 300 + chapters[0].content = "

Chapter 1

" + chapters[0].fetched_at = datetime.now(UTC) + chapters[1].is_stubbed = True + chapters[1].stubbed_at = datetime.now(UTC) + chapters[1].word_count = None + chapters[1].content = None + await db_session.commit() + + resp = await client.post( + f"/api/serials/{serial.id}/volumes/preview", + json={"splits": [{"start": 1, "end": 2, "name": "Alpha"}]}, + ) + assert resp.status_code == 200 + assert resp.json() == [ + { + "start": 1, + "end": 2, + "name": "Alpha", + "chapter_count": 2, + "fetched_chapter_count": 1, + "total_words": 300, + "estimated_pages": 1, + "is_partial": True, + "stubbed_missing_count": 1, + } + ] + + @pytest.mark.asyncio async def test_api_fetch_chapters_status_updates_while_job_runs(client, serial): from app.scrapers.base import ChapterContent @@ -1811,7 +1993,7 @@ async def test_check_serial_for_updates_finds_new_chapters(db_session, serial): ChapterInfo( chapter_number=i, title=f"Chapter {i}", - source_url=f"https://example.com/{i}", + source_url=f"https://royalroad.com/ch/{i}", publish_date=None, ) for i in range(1, 6) # 5 chapters total, serial has 3 @@ -1837,7 +2019,7 @@ async def test_check_serial_for_updates_no_new(db_session, serial): ChapterInfo( chapter_number=i, title=f"Chapter {i}", - source_url=f"https://example.com/{i}", + source_url=f"https://royalroad.com/ch/{i}", publish_date=None, ) for i in range(1, 4) diff --git a/frontend/src/__tests__/ChapterList.test.tsx b/frontend/src/__tests__/ChapterList.test.tsx index be63e03..089cb38 100644 --- a/frontend/src/__tests__/ChapterList.test.tsx +++ b/frontend/src/__tests__/ChapterList.test.tsx @@ -17,6 +17,8 @@ function makeChapter( chapter_number: chapterNumber, title: `Chapter ${chapterNumber}`, source_url: `https://example.com/ch/${chapterNumber}`, + is_stubbed: false, + stubbed_at: null, publish_date: null, word_count: null, estimated_pages: null, diff --git a/frontend/src/__tests__/SerialDetail.test.tsx b/frontend/src/__tests__/SerialDetail.test.tsx index cdc029e..8274aab 100644 --- a/frontend/src/__tests__/SerialDetail.test.tsx +++ b/frontend/src/__tests__/SerialDetail.test.tsx @@ -16,6 +16,8 @@ const SERIAL: WebSerial = { cover_url: 'https://example.com/cover.jpg', status: 'ongoing', total_chapters: 50, + live_chapter_count: 50, + stubbed_chapter_count: 0, last_checked_at: null, last_error: null, created_at: '2026-01-01T00:00:00', diff --git a/frontend/src/__tests__/Serials.test.tsx b/frontend/src/__tests__/Serials.test.tsx index 9c4269c..0e75d8a 100644 --- a/frontend/src/__tests__/Serials.test.tsx +++ b/frontend/src/__tests__/Serials.test.tsx @@ -16,6 +16,8 @@ const SERIALS: object[] = [ cover_url: null, status: 'ongoing', total_chapters: 120, + live_chapter_count: 120, + stubbed_chapter_count: 0, last_checked_at: null, last_error: null, created_at: '2026-01-01T00:00:00', @@ -32,6 +34,8 @@ const SERIALS: object[] = [ cover_url: null, status: 'completed', total_chapters: 200, + live_chapter_count: 200, + stubbed_chapter_count: 0, last_checked_at: null, last_error: null, created_at: '2026-01-02T00:00:00', @@ -337,6 +341,8 @@ const SERIAL_FIXTURE: WebSerial = { cover_url: null, status: 'ongoing', total_chapters: 100, + live_chapter_count: 100, + stubbed_chapter_count: 0, last_checked_at: null, last_error: null, created_at: '2026-01-01T00:00:00', diff --git a/frontend/src/__tests__/VolumeList.test.tsx b/frontend/src/__tests__/VolumeList.test.tsx index c8ebd6c..014cccf 100644 --- a/frontend/src/__tests__/VolumeList.test.tsx +++ b/frontend/src/__tests__/VolumeList.test.tsx @@ -39,6 +39,7 @@ describe('VolumeList', () => { total_words: 560, estimated_pages: 2, is_partial: true, + stubbed_missing_count: 1, }, ], }) as Promise @@ -79,6 +80,7 @@ describe('VolumeList', () => { expect(screen.getByText('~2* pages')).toBeInTheDocument() expect(screen.getByText('560 words')).toBeInTheDocument() expect(screen.getByText('2/3 fetched')).toBeInTheDocument() + expect(screen.getByText('1 stubbed missing')).toBeInTheDocument() expect(previewRequests[previewRequests.length - 1]).toEqual({ splits: [{ start: 1, end: 3 }], }) diff --git a/frontend/src/components/serials/ChapterList.tsx b/frontend/src/components/serials/ChapterList.tsx index 9b5eb1c..5fd95f3 100644 --- a/frontend/src/components/serials/ChapterList.tsx +++ b/frontend/src/components/serials/ChapterList.tsx @@ -454,6 +454,11 @@ export default function ChapterList({

{chapter.title ?? `Chapter ${chapter.chapter_number}`}

+ {chapter.is_stubbed && ( + + STUBBED + + )}

Published {fmtDate(chapter.publish_date)} · Fetched{' '} diff --git a/frontend/src/components/serials/SerialCard.tsx b/frontend/src/components/serials/SerialCard.tsx index 3c2a222..115be9c 100644 --- a/frontend/src/components/serials/SerialCard.tsx +++ b/frontend/src/components/serials/SerialCard.tsx @@ -64,9 +64,15 @@ export default function SerialCard({ serial }: SerialCardProps) { {/* Chapter count */}

- + {serial.total_chapters} ch + {serial.stubbed_chapter_count > 0 && ( + + {serial.live_chapter_count} live · {serial.stubbed_chapter_count}{' '} + stubbed + + )}
diff --git a/frontend/src/components/serials/VolumeList.tsx b/frontend/src/components/serials/VolumeList.tsx index 5fc4efd..e2613fc 100644 --- a/frontend/src/components/serials/VolumeList.tsx +++ b/frontend/src/components/serials/VolumeList.tsx @@ -507,6 +507,11 @@ export default function VolumeList({

{item.fetched_chapter_count}/{item.chapter_count} fetched

+ {item.stubbed_missing_count > 0 && ( +

+ {item.stubbed_missing_count} stubbed missing +

+ )} ))} @@ -516,7 +521,8 @@ export default function VolumeList({

Preview totals use fetched chapter word counts only. Entries marked with `*` are partial because some chapters in the range still need - content. + content. Stubbed missing counts show chapters removed upstream + before they were cached locally.

)} @@ -610,6 +616,14 @@ export default function VolumeList({ )}

+

+ {vol.fetched_chapter_count}/{vol.chapter_count} fetched + {vol.stubbed_missing_count > 0 && ( + + {vol.stubbed_missing_count} stubbed missing + + )} +

{/* Status */} @@ -619,6 +633,11 @@ export default function VolumeList({ STALE )} + {vol.is_partial && ( + + PARTIAL + + )} {vol.generated_at && ( {fmtDate(vol.generated_at)} diff --git a/frontend/src/pages/Home.tsx b/frontend/src/pages/Home.tsx index a2e2ef1..aad3d9b 100644 --- a/frontend/src/pages/Home.tsx +++ b/frontend/src/pages/Home.tsx @@ -333,6 +333,11 @@ function NewChaptersCard({ serial }: { serial: SerialDashboardEntry }) { {serial.fetched_count}/{serial.total_chapters} fetched + {serial.stubbed_chapter_count > 0 && ( + + {serial.stubbed_chapter_count} stubbed + + )}
diff --git a/frontend/src/pages/SerialDetail.tsx b/frontend/src/pages/SerialDetail.tsx index 5b0c07a..628c777 100644 --- a/frontend/src/pages/SerialDetail.tsx +++ b/frontend/src/pages/SerialDetail.tsx @@ -290,6 +290,12 @@ export default function SerialDetail() {

{displaySerial.total_chapters}

+ {displaySerial.stubbed_chapter_count > 0 && ( +

+ {displaySerial.live_chapter_count} live ·{' '} + {displaySerial.stubbed_chapter_count} stubbed +

+ )}
@@ -410,6 +416,8 @@ export default function SerialDetail() { {displaySerial.total_chapters} total + {displaySerial.stubbed_chapter_count > 0 && + ` · ${displaySerial.stubbed_chapter_count} stubbed`}