feat(serials): preserve stubbed chapters

backend/alembic/versions/3c1d7f8a9b21_preserve_stubbed_serial_chapters.py

@@ -0,0 +1,96 @@
+"""preserve stubbed serial chapters
+
+Revision ID: 3c1d7f8a9b21
+Revises: 6305b1bb905b
+Create Date: 2026-03-28 17:35:00.000000
+
+"""
+
+from collections.abc import Sequence
+from urllib.parse import urlparse
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "3c1d7f8a9b21"
+down_revision: str | None = "6305b1bb905b"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _normalize_source_key(url: str) -> str:
+    parsed = urlparse(url)
+    return parsed._replace(fragment="").geturl()
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("web_serials") as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "live_chapter_count",
+                sa.Integer(),
+                nullable=False,
+                server_default="0",
+            )
+        )
+
+    with op.batch_alter_table("serial_chapters") as batch_op:
+        batch_op.add_column(sa.Column("source_key", sa.Text(), nullable=True))
+        batch_op.add_column(
+            sa.Column(
+                "is_stubbed",
+                sa.Boolean(),
+                nullable=False,
+                server_default=sa.false(),
+            )
+        )
+        batch_op.add_column(sa.Column("stubbed_at", sa.DateTime(), nullable=True))
+
+    bind = op.get_bind()
+    chapters = bind.execute(sa.text("SELECT id, source_url FROM serial_chapters")).fetchall()
+    for chapter_id, source_url in chapters:
+        bind.execute(
+            sa.text("UPDATE serial_chapters SET source_key = :source_key WHERE id = :chapter_id"),
+            {
+                "chapter_id": chapter_id,
+                "source_key": _normalize_source_key(source_url),
+            },
+        )
+    op.execute(
+        """
+        UPDATE web_serials
+        SET total_chapters = COALESCE(
+                (SELECT MAX(ch.chapter_number)
+                 FROM serial_chapters AS ch
+                 WHERE ch.serial_id = web_serials.id),
+                0
+            ),
+            live_chapter_count = COALESCE(
+                (SELECT MAX(ch.chapter_number)
+                 FROM serial_chapters AS ch
+                 WHERE ch.serial_id = web_serials.id),
+                0
+            )
+        """
+    )
+
+    with op.batch_alter_table("serial_chapters") as batch_op:
+        batch_op.alter_column("source_key", existing_type=sa.Text(), nullable=False)
+        batch_op.create_index(
+            "ix_serial_chapters_serial_id_source_key",
+            ["serial_id", "source_key"],
+            unique=True,
+        )
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("serial_chapters") as batch_op:
+        batch_op.drop_index("ix_serial_chapters_serial_id_source_key")
+        batch_op.drop_column("stubbed_at")
+        batch_op.drop_column("is_stubbed")
+        batch_op.drop_column("source_key")
+
+    with op.batch_alter_table("web_serials") as batch_op:
+        batch_op.drop_column("live_chapter_count")

backend/app/models/serial.py

@@ -23,6 +23,7 @@ class WebSerial(Base):
         Text, default="ongoing", nullable=False
     )  # ongoing|completed|paused|error
     total_chapters: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    live_chapter_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     last_checked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
     last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
     source_metadata: Mapped[str | None] = mapped_column(Text, nullable=True)  # JSON blob
@@ -42,6 +43,10 @@ class WebSerial(Base):
     )
     series: Mapped["Series | None"] = relationship("Series")  # type: ignore[name-defined]  # noqa: F821
 
+    @property
+    def stubbed_chapter_count(self) -> int:
+        return max(self.total_chapters - self.live_chapter_count, 0)
+
 
 class SerialChapter(Base):
     __tablename__ = "serial_chapters"
@@ -52,13 +57,20 @@ class SerialChapter(Base):
             "chapter_number",
             unique=True,
         ),
+        Index(
+            "ix_serial_chapters_serial_id_source_key",
+            "serial_id",
+            "source_key",
+            unique=True,
+        ),
     )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
     serial_id: Mapped[int] = mapped_column(
         Integer, ForeignKey("web_serials.id", ondelete="CASCADE"), nullable=False
     )
     chapter_number: Mapped[int] = mapped_column(Integer, nullable=False)  # 1-based
+    source_key: Mapped[str] = mapped_column(Text, nullable=False)
     title: Mapped[str | None] = mapped_column(Text, nullable=True)
     source_url: Mapped[str] = mapped_column(Text, nullable=False)
     publish_date: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
@@ -67,6 +79,8 @@ class SerialChapter(Base):
     )  # cleaned HTML, null until fetched
     word_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
     fetched_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    is_stubbed: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    stubbed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
 
     serial: Mapped["WebSerial"] = relationship("WebSerial", back_populates="chapters")
 

backend/app/routers/serials.py

@@ -43,7 +43,7 @@
     generate_volume,
     get_chapter_fetch_status,
     get_serial,
-    get_volume_word_counts,
+    get_volume_metrics,
     list_chapter_responses,
     list_serials,
     list_serials_for_dashboard,
@@ -66,15 +66,22 @@
 WORDS_PER_PAGE = 280
 
 
-def _enrich_volumes(volumes: list[object], word_counts: dict[int, int]) -> list[VolumeResponse]:
-    """Build VolumeResponse list with estimated_pages and total_words."""
+def _enrich_volumes(volumes: list[object], metrics: dict[int, object]) -> list[VolumeResponse]:
+    """Build VolumeResponse list with derived volume metrics."""
     result: list[VolumeResponse] = []
     for v in volumes:
         resp = VolumeResponse.model_validate(v)
-        words = word_counts.get(resp.id)
-        if words is not None and words > 0:
-            resp.total_words = words
-            resp.estimated_pages = max(1, words // WORDS_PER_PAGE)
+        metric = metrics.get(resp.id)
+        if metric is not None:
+            total_words = int(getattr(metric, "total_words"))
+            resp.total_words = total_words
+            resp.estimated_pages = (
+                max(1, total_words // WORDS_PER_PAGE) if total_words > 0 else None
+            )
+            resp.chapter_count = int(getattr(metric, "chapter_count"))
+            resp.fetched_chapter_count = int(getattr(metric, "fetched_chapter_count"))
+            resp.is_partial = bool(getattr(metric, "is_partial"))
+            resp.stubbed_missing_count = int(getattr(metric, "stubbed_missing_count"))
         result.append(resp)
     return result
 
@@ -342,8 +349,8 @@ async def configure_volumes_endpoint(
     except Exception as exc:
         log.exception("Failed to configure volumes for serial %d", serial_id)
         raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc))
-    word_counts = await get_volume_word_counts(session, serial_id)
-    return _enrich_volumes(volumes, word_counts)
+    metrics = await get_volume_metrics(session, serial_id)
+    return _enrich_volumes(volumes, metrics)
 
 
 @router.post(
@@ -356,8 +363,8 @@ async def auto_split_volumes_endpoint(
         volumes = await auto_split_volumes(session, serial_id, body)
     except SerialNotFound as exc:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
-    word_counts = await get_volume_word_counts(session, serial_id)
-    return _enrich_volumes(volumes, word_counts)
+    metrics = await get_volume_metrics(session, serial_id)
+    return _enrich_volumes(volumes, metrics)
 
 
 @router.post("/serials/{serial_id}/volumes/add", response_model=VolumeResponse, status_code=201)
@@ -368,8 +375,8 @@ async def add_single_volume_endpoint(
         vol = await add_single_volume(session, serial_id, body)
     except SerialNotFound as exc:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
-    word_counts = await get_volume_word_counts(session, serial_id)
-    return _enrich_volumes([vol], word_counts)[0]
+    metrics = await get_volume_metrics(session, serial_id)
+    return _enrich_volumes([vol], metrics)[0]
 
 
 @router.get("/serials/{serial_id}/volumes", response_model=list[VolumeResponse])
@@ -378,8 +385,8 @@ async def list_volumes_endpoint(serial_id: int, session: AsyncSession = Depends(
         volumes = await list_volumes(session, serial_id)
     except SerialNotFound as exc:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
-    word_counts = await get_volume_word_counts(session, serial_id)
-    return _enrich_volumes(volumes, word_counts)
+    metrics = await get_volume_metrics(session, serial_id)
+    return _enrich_volumes(volumes, metrics)
 
 
 @router.patch("/serials/{serial_id}/volumes/{volume_id}", response_model=VolumeResponse)
@@ -393,8 +400,8 @@ async def update_volume_endpoint(
         vol = await update_volume(session, serial_id, volume_id, body)
     except SerialNotFound as exc:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
-    word_counts = await get_volume_word_counts(session, serial_id)
-    return _enrich_volumes([vol], word_counts)[0]
+    metrics = await get_volume_metrics(session, serial_id)
+    return _enrich_volumes([vol], metrics)[0]
 
 
 @router.delete("/serials/{serial_id}/volumes/{volume_id}", status_code=status.HTTP_204_NO_CONTENT)
@@ -426,8 +433,8 @@ async def upload_volume_cover_endpoint(
         vol = await upload_volume_cover(session, serial_id, volume_id, content, suffix)
     except SerialNotFound as exc:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
-    word_counts = await get_volume_word_counts(session, serial_id)
-    return _enrich_volumes([vol], word_counts)[0]
+    metrics = await get_volume_metrics(session, serial_id)
+    return _enrich_volumes([vol], metrics)[0]
 
 
 @router.post("/serials/{serial_id}/volumes/{volume_id}/generate", response_model=VolumeResponse)
@@ -447,8 +454,8 @@ async def generate_volume_endpoint(
     except Exception as exc:
         log.exception("Unexpected error generating volume %d for serial %d", volume_id, serial_id)
         raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc))
-    word_counts = await get_volume_word_counts(session, serial_id)
-    return _enrich_volumes([vol], word_counts)[0]
+    metrics = await get_volume_metrics(session, serial_id)
+    return _enrich_volumes([vol], metrics)[0]
 
 
 @router.post("/serials/{serial_id}/volumes/generate-all", response_model=list[VolumeResponse])
@@ -461,8 +468,8 @@ async def generate_all_volumes_endpoint(
         volumes = await generate_all_volumes(session, serial_id, shelf_id)
     except SerialNotFound as exc:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
-    word_counts = await get_volume_word_counts(session, serial_id)
-    return _enrich_volumes(volumes, word_counts)
+    metrics = await get_volume_metrics(session, serial_id)
+    return _enrich_volumes(volumes, metrics)
 
 
 @router.post("/serials/{serial_id}/volumes/{volume_id}/rebuild", response_model=VolumeResponse)
@@ -477,5 +484,5 @@ async def rebuild_volume_endpoint(
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
     except VolumeGenerationError as exc:
         raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc))
-    word_counts = await get_volume_word_counts(session, serial_id)
-    return _enrich_volumes([vol], word_counts)[0]
+    metrics = await get_volume_metrics(session, serial_id)
+    return _enrich_volumes([vol], metrics)[0]

backend/app/schemas/serial.py

@@ -30,6 +30,8 @@ class SerialResponse(BaseModel):
     cover_url: str | None
     status: str
     total_chapters: int
+    live_chapter_count: int
+    stubbed_chapter_count: int
     last_checked_at: datetime | None
     last_error: str | None
     created_at: datetime
@@ -43,6 +45,8 @@ class SerialDashboardResponse(BaseModel):
     cover_path: str | None
     status: str
     total_chapters: int
+    live_chapter_count: int
+    stubbed_chapter_count: int
     fetched_count: int
     new_chapter_count: int
     latest_chapter_title: str | None
@@ -57,6 +61,8 @@ class ChapterResponse(BaseModel):
     chapter_number: int
     title: str | None
     source_url: str
+    is_stubbed: bool = False
+    stubbed_at: datetime | None = None
     publish_date: datetime | None
     word_count: int | None
     estimated_pages: int | None = None
@@ -85,6 +91,8 @@ def from_orm(
             chapter_number=obj.chapter_number,
             title=obj.title,
             source_url=obj.source_url,
+            is_stubbed=obj.is_stubbed,
+            stubbed_at=obj.stubbed_at,
             publish_date=obj.publish_date,
             word_count=obj.word_count,
             estimated_pages=estimated_pages,
@@ -171,6 +179,10 @@ class VolumeResponse(BaseModel):
     chapter_end: int
     generated_at: datetime | None
     is_stale: bool
+    chapter_count: int = 0
+    fetched_chapter_count: int = 0
+    is_partial: bool = False
+    stubbed_missing_count: int = 0
     estimated_pages: int | None = None
     total_words: int | None = None
 
@@ -184,3 +196,4 @@ class VolumePreviewResponse(BaseModel):
     total_words: int = 0
     estimated_pages: int | None = None
     is_partial: bool = False
+    stubbed_missing_count: int = 0