From d081dd25e6aa17020814cb428336175dd2a804ea Mon Sep 17 00:00:00 2001 From: Paul Rimmer Date: Tue, 22 Sep 2026 15:26:57 +0700 Subject: [PATCH 1/5] Add the ability to process more than 1 page of AA search results, similar to ABB --- docker-compose.dev.yml | 2 +- shelfmark/config/settings.py | 12 ++ .../direct_download/annas_archive.py | 192 +++++++++++++++--- src/frontend/src/components/ReleaseModal.tsx | 18 +- .../src/components/ResultsSection.tsx | 2 +- .../components/resultsViews/CompactView.tsx | 1 + .../src/components/resultsViews/ListView.tsx | 9 +- tests/direct_download/test_search_queries.py | 26 ++- 8 files changed, 213 insertions(+), 49 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 57927954b..f42fc38ce 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,7 +2,7 @@ services: shelfmark-dev: extends: - file: ./compose/docker-compose.yml + file: ./compose/shelfmark.yml service: shelfmark build: context: . diff --git a/shelfmark/config/settings.py b/shelfmark/config/settings.py index 1ee760a1a..226f83999 100644 --- a/shelfmark/config/settings.py +++ b/shelfmark/config/settings.py @@ -1683,6 +1683,18 @@ def download_source_settings() -> list[SettingsField]: min_value=30, max_value=1800, ), + NumberField( + key="AA_PAGE_LIMIT", + label="Max Anna's Archive Results Pages", + description=( + "Maximum number of results pages to fetch per manual search. " + "Each page returns 50 books (up to 500 total). " + "Set to 1 for first-page only, up to 10 for full 500 results." + ), + default=1, + min_value=1, + max_value=10, + ), HeadingField( key="content_type_routing_heading", title="Content-Type Routing", diff --git a/shelfmark/release_sources/direct_download/annas_archive.py b/shelfmark/release_sources/direct_download/annas_archive.py index d8eb31c2d..aca6cd883 100644 --- a/shelfmark/release_sources/direct_download/annas_archive.py +++ b/shelfmark/release_sources/direct_download/annas_archive.py @@ -48,6 +48,20 @@ book_matches_requested_languages as _book_matches_requested_languages, ) +# --------------------------------------------------------------------------- +# Config helpers +# --------------------------------------------------------------------------- + + +def _coerce_positive_int(value: object, default: int) -> int: + """Return a positive integer config value or the provided default.""" + if isinstance(value, bool): + return default + if isinstance(value, int) and value > 0: + return value + return default + + if TYPE_CHECKING: from collections.abc import Callable, Iterable, Iterator from pathlib import Path @@ -707,12 +721,18 @@ def _extract_total_results_from_html(html: str) -> int | str | None: return None -def search_books(query: str, filters: SearchFilters) -> tuple[list[BrowseRecord], int | str | None]: - """Search for books matching the query. +def search_books( + query: str, + filters: SearchFilters, + *, + page: int = 1, +) -> tuple[list[BrowseRecord], int | str | None]: + """Search for books matching the query on a single page. Args: query: Search term (ISBN, title, author, etc.) filters: Search filters (language, format, content type, etc.) + page: Page number to fetch (default 1). AA returns 50 results per page. Returns: Tuple of (List[BrowseRecord], total_count | None): Matching books and total result count @@ -762,7 +782,7 @@ def search_books(query: str, filters: SearchFilters) -> tuple[list[BrowseRecord] url = ( f"{network.get_aa_base_url()}" - f"/search?index=&page=1&display=table" + f"/search?index=&page={page}&display=table" f"&acc=aa_download&acc=external_download" f"&ext={'&ext='.join(formats_to_use)}" f"&q={query_html}" @@ -808,10 +828,6 @@ def search_books(query: str, filters: SearchFilters) -> tuple[list[BrowseRecord] ) ) - # Fetch download counts for all results in batch - if books: - _enrich_search_results_with_downloads(books) - return (books, total_count) @@ -1910,30 +1926,77 @@ def _search_books_with_language_fallback( query: str, filters: SearchFilters, *, - search_label: str, - ) -> list[BrowseRecord]: - """Retry AA queries without a language filter when filtered search returns nothing.""" + max_pages: int, + ) -> tuple[list[BrowseRecord], int | str | None, SearchFilters]: + """Retry AA queries without a language filter when filtered search returns nothing. + + Always returns ``(results, total_count, filters_used)`` so the caller can + paginate with the correct (possibly modified) filters. + """ results, total_count = self._search_books(query, filters) if results or not filters.lang: self._total_results = total_count - return results + return results, total_count, filters - logger.debug( - "No %s results with langs=%s, retrying without language filter", - search_label, - filters.lang, - ) - results, total_count = self._search_books(query, replace(filters, lang=None)) + filters_used = replace(filters, lang=None) + results, total_count = self._search_books(query, filters_used) self._total_results = total_count - return results + return results, total_count, filters_used + + def _search_books_paginated( + self, + query: str, + filters: SearchFilters, + *, + total_pages: int, + ) -> list[BrowseRecord]: + """Fetch multiple pages of results for a manual search. + + Returns a deduplicated list across all pages. Download counts are + enriched once after the last page is fetched. + """ + seen_ids: set[str] = set() + all_results: list[BrowseRecord] = [] + supported_formats = get_supported_formats() + + for page_num in range(1, total_pages + 1): + if search_deadline.expired(): + break + + books, _ = self._search_books(query, filters, page=page_num) + + for bi in books: + if bi.id not in seen_ids: + seen_ids.add(bi.id) + all_results.append(bi) + + # Stop if this page returned fewer than 50 results (last page) + if len(books) < 50: + break + + # Single enrichment pass after all pages are collected + if all_results: + _enrich_search_results_with_downloads(all_results) + + all_results.sort( + key=lambda x: ( + supported_formats.index(x.format) + if x.format in supported_formats + else len(supported_formats) + ) + ) + + return all_results def _search_books( self, query: str, filters: SearchFilters, + *, + page: int = 1, ) -> tuple[list[BrowseRecord], int | str | None]: """Call search_books and capture the total result count.""" - return search_books(query, filters) + return search_books(query, filters, page=page) def search( self, @@ -1986,11 +2049,27 @@ def _search( ) filters = plan.source_filters or SearchFilters() filters.lang = lang_filter if lang_filter is not None else (filters.lang or []) - results = self._search_books_with_language_fallback( - query, filters, search_label="manual" - ) + result = self._search_books_with_language_fallback(query, filters, max_pages=10) self._last_search_type = "manual" if query else "title_author" - return results + + # _search_books_with_language_fallback always returns + # (results, total_count, filters_used) when max_pages is set. + # Paginate through all pages of results using the filters that + # actually produced results (may have lang=None after fallback). + results, total_count, filters_used = result # type: ignore[misc] + self._total_results = total_count + + # Calculate how many pages to fetch, capped by user config + aa_page_limit = _coerce_positive_int(config.get("AA_PAGE_LIMIT", 1), 1) + if isinstance(total_count, str): + # Capped at 500+ + max_pages = min(aa_page_limit, 10) # 500 / 50 per page + elif total_count is not None: + max_pages = min(aa_page_limit, (total_count + 49) // 50) # ceil division + else: + max_pages = min(aa_page_limit, 10) # reasonable default when total is unknown + + return self._search_books_paginated(query, filters_used, total_pages=max_pages) # ISBN search first (unless expand_search requested) if plan.manual_query: @@ -1999,7 +2078,6 @@ def _search( if not expand_search: isbn = plan.isbn_candidates[0] if plan.isbn_candidates else None if isbn: - logger.debug("Searching direct_download: isbn='%s', langs=%s", isbn, lang_filter) filters = SearchFilters(isbn=[isbn]) filters.lang = lang_filter if lang_filter is not None else [] try: @@ -2037,12 +2115,43 @@ def _search( logger.debug("Searching direct_download: title_author='%s', langs=%s", query, langs) filters = SearchFilters(lang=langs if langs is not None else []) try: - books, title_total = search_books(query, filters) + # Fetch first page to get total count for pagination + first_page, title_total = search_books(query, filters) self._total_results = title_total - for bi in books: + + # Add first page results + for bi in first_page: if bi.id not in seen_ids: seen_ids.add(bi.id) all_results.append(bi) + + # Only paginate if the first page had 50+ results (more pages may exist) + if len(first_page) >= 50: + # Calculate how many pages to fetch, capped by user config + aa_page_limit = _coerce_positive_int(config.get("AA_PAGE_LIMIT", 1), 1) + if isinstance(title_total, str): + max_pages = min(aa_page_limit, 10) # 500 / 50 per page + elif title_total is not None: + max_pages = min(aa_page_limit, (title_total + 49) // 50) # ceil division + else: + max_pages = min(aa_page_limit, 10) # reasonable default + + # Fetch additional pages if needed + if max_pages > 1: + for page_num in range(2, max_pages + 1): + if search_deadline.expired(): + logger.info( + "Release search budget spent; stopping pagination at page %d", + page_num, + ) + break + books, _ = self._search_books(query, filters, page=page_num) + for bi in books: + if bi.id not in seen_ids: + seen_ids.add(bi.id) + all_results.append(bi) + if len(books) < 50: + break except SearchUnavailableError: raise except Exception: @@ -2053,24 +2162,41 @@ def _search( and any(langs for _, langs in searches) and not search_deadline.expired() ): - logger.debug( - "No title+author results with language filter, retrying without language filter" - ) for title, _langs in searches: query = f"{title} {author}".strip() if not query: continue if search_deadline.expired(): - logger.info("Release search budget spent; skipping remaining retries") break - logger.debug("Searching direct_download: title_author='%s', langs=[]", query) try: - books, _ = search_books(query, SearchFilters()) - for bi in books: + # Fetch first page to get total count for pagination + first_page, _ = search_books(query, SearchFilters()) + + # Add first page results + for bi in first_page: if bi.id not in seen_ids: seen_ids.add(bi.id) all_results.append(bi) + + # Only paginate if the first page had 50+ results (more pages may exist) + if len(first_page) >= 50: + # Calculate how many pages to fetch, capped by user config + aa_page_limit = _coerce_positive_int(config.get("AA_PAGE_LIMIT", 1), 1) + max_pages = min(aa_page_limit, 10) # reasonable default + + # Fetch additional pages if needed + if max_pages > 1: + for page_num in range(2, max_pages + 1): + if search_deadline.expired(): + break + books, _ = self._search_books(query, SearchFilters(), page=page_num) + for bi in books: + if bi.id not in seen_ids: + seen_ids.add(bi.id) + all_results.append(bi) + if len(books) < 50: + break except SearchUnavailableError: raise except Exception: diff --git a/src/frontend/src/components/ReleaseModal.tsx b/src/frontend/src/components/ReleaseModal.tsx index 05813245c..91fe44ede 100644 --- a/src/frontend/src/components/ReleaseModal.tsx +++ b/src/frontend/src/components/ReleaseModal.tsx @@ -438,10 +438,10 @@ const ReleaseRow = ({ // Build grid template based on whether leading cell is shown const desktopGridTemplate = showLeadingCell - ? `auto ${gridTemplate} auto` - : `${gridTemplate} auto`; + ? `auto 28px ${gridTemplate} auto` + : `28px ${gridTemplate} auto`; - const mobileGridTemplate = showLeadingCell ? 'auto 1fr auto' : '1fr auto'; + const mobileGridTemplate = showLeadingCell ? 'auto 28px 1fr auto' : '28px 1fr auto'; const handleRowClick = selectionMode && onSelect ? onSelect : undefined; const handleRowKeyDown: KeyboardEventHandler | undefined = @@ -483,6 +483,11 @@ const ReleaseRow = ({ {/* Leading cell: Thumbnail, Badge, or nothing */} {showLeadingCell && } + {/* Release number */} + + {index + 1} + + {/* Fixed: Title and author */}

@@ -530,6 +535,11 @@ const ReleaseRow = ({ {/* Leading cell: Thumbnail, Badge, or nothing */} {showLeadingCell && } + {/* Release number */} + + {index + 1} + +

{/* Title and author on same line */}

@@ -1828,7 +1838,7 @@ const ReleaseModalSession = ({ ); } - const shownEnd = Math.min(totalCountNum, 50); + const shownEnd = Math.min(totalCountNum, filteredReleases.length); const totalStr = isCapped ? '500+' : String(totalCountNum); return ( diff --git a/src/frontend/src/components/ResultsSection.tsx b/src/frontend/src/components/ResultsSection.tsx index 8666348db..ad27642ef 100644 --- a/src/frontend/src/components/ResultsSection.tsx +++ b/src/frontend/src/components/ResultsSection.tsx @@ -136,7 +136,7 @@ export const ResultsSection = ({ ); } - const shownEnd = Math.min(totalCount, 50); + const shownEnd = Math.min(totalCount, books.length); const totalStr = isCapped ? '500+' : String(totalCount); return ( diff --git a/src/frontend/src/components/resultsViews/CompactView.tsx b/src/frontend/src/components/resultsViews/CompactView.tsx index 217d9ecbb..c52c5d2f4 100644 --- a/src/frontend/src/components/resultsViews/CompactView.tsx +++ b/src/frontend/src/components/resultsViews/CompactView.tsx @@ -21,6 +21,7 @@ interface CompactViewProps { showDetailsButton?: boolean; animationDelay?: number; showSeriesPosition?: boolean; + index?: number; onShowToast?: (message: string, type: 'success' | 'error' | 'info') => void; } diff --git a/src/frontend/src/components/resultsViews/ListView.tsx b/src/frontend/src/components/resultsViews/ListView.tsx index f2e68ebe9..7c83e3fb5 100644 --- a/src/frontend/src/components/resultsViews/ListView.tsx +++ b/src/frontend/src/components/resultsViews/ListView.tsx @@ -167,8 +167,8 @@ export const ListView = ({

{/* Thumbnail */} @@ -180,6 +180,11 @@ export const ListView = ({ />
+ {/* Release number */} + + {index + 1} + + {/* Title and Author */}

Date: Tue, 22 Sep 2026 15:58:08 +0700 Subject: [PATCH 2/5] Fix the compose file that was incorrectly commited on the branch --- docker-compose.dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f42fc38ce..57927954b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,7 +2,7 @@ services: shelfmark-dev: extends: - file: ./compose/shelfmark.yml + file: ./compose/docker-compose.yml service: shelfmark build: context: . From 4de5e7334c5ff9e5880e3ad814fb18bce90bab18 Mon Sep 17 00:00:00 2001 From: Paul Rimmer Date: Mon, 28 Sep 2026 14:20:15 +0700 Subject: [PATCH 3/5] Eliminate redundant page 1 fetches, fix pagination with language filtering amd add timing debug logs --- shelfmark/download/warmup.py | 2 +- shelfmark/main.py | 2 + .../direct_download/annas_archive.py | 156 +++++++++++++++--- .../release_sources/direct_download/source.py | 6 + src/frontend/src/App.tsx | 2 + src/frontend/src/components/ReleaseModal.tsx | 11 +- .../src/components/ResultsSection.tsx | 12 +- src/frontend/src/hooks/useSearch.ts | 9 +- src/frontend/src/services/api.ts | 7 +- src/frontend/src/types/index.ts | 1 + tests/direct_download/test_search_budget.py | 4 +- tests/direct_download/test_search_queries.py | 16 +- 12 files changed, 183 insertions(+), 45 deletions(-) diff --git a/shelfmark/download/warmup.py b/shelfmark/download/warmup.py index de6115bbe..adce6ca6e 100644 --- a/shelfmark/download/warmup.py +++ b/shelfmark/download/warmup.py @@ -114,7 +114,7 @@ def run_warmup() -> bool: from shelfmark.core.models import SearchFilters from shelfmark.release_sources.direct_download import search_books - results, _ = search_books(query, SearchFilters()) + results, _, _ = search_books(query, SearchFilters()) except Exception: # Broad by design: a warm-up must never take the app down, and the source # raises everything from network errors to parse failures. diff --git a/shelfmark/main.py b/shelfmark/main.py index 40d0b3bf7..c49902978 100644 --- a/shelfmark/main.py +++ b/shelfmark/main.py @@ -3173,6 +3173,8 @@ def _search_source_releases( info["search_type"] = source_instance.last_search_type if hasattr(source_instance, "total_results"): info["total_results"] = source_instance.total_results + if hasattr(source_instance, "filtered_out_count"): + info["filtered_out_count"] = source_instance.filtered_out_count if info: search_info[source_name] = info diff --git a/shelfmark/release_sources/direct_download/annas_archive.py b/shelfmark/release_sources/direct_download/annas_archive.py index aca6cd883..2f5b0bfd7 100644 --- a/shelfmark/release_sources/direct_download/annas_archive.py +++ b/shelfmark/release_sources/direct_download/annas_archive.py @@ -726,7 +726,7 @@ def search_books( filters: SearchFilters, *, page: int = 1, -) -> tuple[list[BrowseRecord], int | str | None]: +) -> tuple[list[BrowseRecord], int | str | None, int]: """Search for books matching the query on a single page. Args: @@ -735,13 +735,16 @@ def search_books( page: Page number to fetch (default 1). AA returns 50 results per page. Returns: - Tuple of (List[BrowseRecord], total_count | None): Matching books and total result count + Tuple of (books, total_count, raw_page_count) where *raw_page_count* is + the number of results returned by AA *before* language filtering, so + callers can correctly detect the last page. Raises: SearchUnavailableError: If Anna's Archive cannot be reached Exception: If parsing fails """ + _t0 = time.perf_counter() query_html = quote(query) if filters.isbn: @@ -789,14 +792,16 @@ def search_books( f"{filters_query}" ) + _fetch_t0 = time.perf_counter() # AA gates /search behind a DDoS-Guard JS challenge, which every mirror shares. Rotating # to another mirror only collects another 403, so let the bypasser solve it. html, tbody = _fetch_search_table(url, selector) + _fetch_elapsed = time.perf_counter() - _fetch_t0 total_count = _extract_total_results_from_html(html) if tbody is not None else None if tbody is None: if "No files found." in html: logger.info("No books found for query: %s", query) - return ([], None) + return ([], None, 0) logger.warning("No results table found for query: %s", query) msg = "No books found. Please try another query." raise RuntimeError(msg) @@ -804,6 +809,7 @@ def search_books( msg = f"Expected results table tag, got {type(tbody).__name__}" raise TypeError(msg) + _parse_t0 = time.perf_counter() books = parse_search_page( tbody, filters, @@ -814,6 +820,16 @@ def search_books( # (which skips &lang=) needs a local language filter. filter_languages=False, ) + _parse_elapsed = time.perf_counter() - _parse_t0 + logger.debug( + "Search parse completed for query=%r page=%d: %d records in %.2fs", + query, + page, + len(books), + _parse_elapsed, + ) + + raw_page_count = len(books) if path_language_enabled and requested_langs: books = [b for b in books if _book_matches_requested_languages(b.language, requested_langs)] @@ -828,7 +844,18 @@ def search_books( ) ) - return (books, total_count) + _total_elapsed = time.perf_counter() - _t0 + logger.info( + "Search complete for query=%r page=%d: %d records (fetch=%.2fs, parse=%.2fs, total=%.2fs)", + query, + page, + len(books), + _fetch_elapsed, + _parse_elapsed, + _total_elapsed, + ) + + return (books, total_count, raw_page_count) def _fetch_download_count_inline(book_id: str) -> int | None: @@ -855,6 +882,11 @@ def _enrich_search_results_with_downloads(books: list[BrowseRecord]) -> None: if not book_ids: return + _enrich_t0 = time.perf_counter() + logger.debug( + "Fetching download counts for %d books via inline_info", len(book_ids), + ) + # Fetch counts in parallel using the inline_info API (cheaper than summary) counts: dict[str, int] = {} with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: @@ -868,6 +900,14 @@ def _enrich_search_results_with_downloads(books: list[BrowseRecord]) -> None: except Exception: logger.debug("Failed to fetch download count for %s", bid, exc_info=True) + _enrich_elapsed = time.perf_counter() - _enrich_t0 + logger.debug( + "Download count enrichment complete: %d/%d books resolved in %.2fs", + len(counts), + len(books), + _enrich_elapsed, + ) + # Add counts to each record's info for book in books: if book.id in counts: @@ -1882,6 +1922,7 @@ class AnnasArchiveProvider: def __init__(self) -> None: self._last_search_type = "title_author" self._total_results: int | str | None = None + self._filtered_out_count: int = 0 @property def last_search_type(self) -> str: @@ -1891,6 +1932,10 @@ def last_search_type(self) -> str: def total_results(self) -> int | str | None: return self._total_results + @property + def filtered_out_count(self) -> int: + return self._filtered_out_count + def is_enabled(self) -> bool: from shelfmark.core import mirrors @@ -1933,13 +1978,14 @@ def _search_books_with_language_fallback( Always returns ``(results, total_count, filters_used)`` so the caller can paginate with the correct (possibly modified) filters. """ - results, total_count = self._search_books(query, filters) + results, total_count, first_page_raw = self._search_books(query, filters) if results or not filters.lang: self._total_results = total_count + self._first_page_raw_count = first_page_raw return results, total_count, filters filters_used = replace(filters, lang=None) - results, total_count = self._search_books(query, filters_used) + results, total_count, _ = self._search_books(query, filters_used) self._total_results = total_count return results, total_count, filters_used @@ -1949,31 +1995,61 @@ def _search_books_paginated( filters: SearchFilters, *, total_pages: int, + first_page_results: list[BrowseRecord] | None = None, ) -> list[BrowseRecord]: """Fetch multiple pages of results for a manual search. - Returns a deduplicated list across all pages. Download counts are + If *first_page_results* is provided, page 1 is skipped (already fetched) + and those results are used as the starting point. Returns a deduplicated + list across all pages. Download counts are enriched once after the last page is fetched. """ seen_ids: set[str] = set() - all_results: list[BrowseRecord] = [] + all_results: list[BrowseRecord] = ( + list(first_page_results) if first_page_results else [] + ) + if first_page_results: + for bi in first_page_results: + seen_ids.add(bi.id) supported_formats = get_supported_formats() + _paginated_t0 = time.perf_counter() - for page_num in range(1, total_pages + 1): + # Track cumulative count of books filtered out by language + filtered_out = 0 + if first_page_results and hasattr(self, "_first_page_raw_count"): + filtered_out += self._first_page_raw_count - len(first_page_results) + + start_page = 2 if first_page_results else 1 + logger.debug("Manual search: fetching pages %d-%d", start_page, total_pages) + + for page_num in range(start_page, total_pages + 1): if search_deadline.expired(): break - books, _ = self._search_books(query, filters, page=page_num) + books, _, raw_count = self._search_books(query, filters, page=page_num) + _page_elapsed = time.perf_counter() - _paginated_t0 for bi in books: if bi.id not in seen_ids: seen_ids.add(bi.id) all_results.append(bi) - # Stop if this page returned fewer than 50 results (last page) - if len(books) < 50: + filtered_out += raw_count - len(books) + + logger.debug( + "Manual search page=%d: %d records (raw=%d) in %.2fs", + page_num, + len(books), + raw_count, + _page_elapsed, + ) + + # Stop only if this page returned zero raw results (genuinely exhausted) + if raw_count == 0: break + self._filtered_out_count = filtered_out + # Single enrichment pass after all pages are collected if all_results: _enrich_search_results_with_downloads(all_results) @@ -1994,8 +2070,13 @@ def _search_books( filters: SearchFilters, *, page: int = 1, - ) -> tuple[list[BrowseRecord], int | str | None]: - """Call search_books and capture the total result count.""" + ) -> tuple[list[BrowseRecord], int | str | None, int]: + """Call search_books and capture the total and raw page result counts. + + Returns ``(books, total_count, raw_page_count)`` where *raw_page_count* + is the number of results returned by AA *before* language filtering, so + callers can correctly detect the last page. + """ return search_books(query, filters, page=page) def search( @@ -2036,6 +2117,7 @@ def _search( content_type: Ignored - Direct download uses format filtering instead """ + _search_t0 = time.perf_counter() ensure_available() lang_filter = plan.languages @@ -2069,7 +2151,15 @@ def _search( else: max_pages = min(aa_page_limit, 10) # reasonable default when total is unknown - return self._search_books_paginated(query, filters_used, total_pages=max_pages) + results = self._search_books_paginated( + query, filters_used, total_pages=max_pages, + first_page_results=results, + ) + _elapsed = time.perf_counter() - _search_t0 + logger.info( + "AA provider search complete: %d records in %.2fs", len(results), _elapsed, + ) + return results # ISBN search first (unless expand_search requested) if plan.manual_query: @@ -2081,11 +2171,16 @@ def _search( filters = SearchFilters(isbn=[isbn]) filters.lang = lang_filter if lang_filter is not None else [] try: - results, isbn_total = search_books(isbn, filters) + results, isbn_total, isbn_raw = search_books(isbn, filters) + _elapsed = time.perf_counter() - _search_t0 if results: logger.info("Found %s releases via ISBN", len(results)) self._last_search_type = "isbn" self._total_results = isbn_total + self._filtered_out_count = isbn_raw - len(results) + logger.info( + "AA provider search complete: %d records in %.2fs", len(results), _elapsed, + ) return results logger.debug("No ISBN results, falling back to title+author") except SearchUnavailableError: @@ -2100,6 +2195,7 @@ def _search( # Execute searches with deduplication seen_ids: set = set() all_results: list[BrowseRecord] = [] + filtered_out = 0 for title, langs in searches: query = f"{title} {author}".strip() @@ -2116,7 +2212,7 @@ def _search( filters = SearchFilters(lang=langs if langs is not None else []) try: # Fetch first page to get total count for pagination - first_page, title_total = search_books(query, filters) + first_page, title_total, first_page_raw = search_books(query, filters) self._total_results = title_total # Add first page results @@ -2125,8 +2221,10 @@ def _search( seen_ids.add(bi.id) all_results.append(bi) - # Only paginate if the first page had 50+ results (more pages may exist) - if len(first_page) >= 50: + filtered_out += first_page_raw - len(first_page) + + # Only paginate if the first page had any results (more pages may exist) + if len(first_page) > 0: # Calculate how many pages to fetch, capped by user config aa_page_limit = _coerce_positive_int(config.get("AA_PAGE_LIMIT", 1), 1) if isinstance(title_total, str): @@ -2145,12 +2243,13 @@ def _search( page_num, ) break - books, _ = self._search_books(query, filters, page=page_num) + books, _, raw_count = self._search_books(query, filters, page=page_num) for bi in books: if bi.id not in seen_ids: seen_ids.add(bi.id) all_results.append(bi) - if len(books) < 50: + filtered_out += raw_count - len(books) + if raw_count == 0: break except SearchUnavailableError: raise @@ -2171,7 +2270,7 @@ def _search( try: # Fetch first page to get total count for pagination - first_page, _ = search_books(query, SearchFilters()) + first_page, _, _ = search_books(query, SearchFilters()) # Add first page results for bi in first_page: @@ -2179,8 +2278,8 @@ def _search( seen_ids.add(bi.id) all_results.append(bi) - # Only paginate if the first page had 50+ results (more pages may exist) - if len(first_page) >= 50: + # Only paginate if the first page had any results (more pages may exist) + if len(first_page) > 0: # Calculate how many pages to fetch, capped by user config aa_page_limit = _coerce_positive_int(config.get("AA_PAGE_LIMIT", 1), 1) max_pages = min(aa_page_limit, 10) # reasonable default @@ -2190,16 +2289,21 @@ def _search( for page_num in range(2, max_pages + 1): if search_deadline.expired(): break - books, _ = self._search_books(query, SearchFilters(), page=page_num) + books, _, raw_count = self._search_books(query, SearchFilters(), page=page_num) for bi in books: if bi.id not in seen_ids: seen_ids.add(bi.id) all_results.append(bi) - if len(books) < 50: + if raw_count == 0: break except SearchUnavailableError: raise except Exception: logger.exception("Search error") + self._filtered_out_count = filtered_out + _elapsed = time.perf_counter() - _search_t0 + logger.info( + "AA provider search complete: %d records in %.2fs", len(all_results), _elapsed, + ) return all_results diff --git a/shelfmark/release_sources/direct_download/source.py b/shelfmark/release_sources/direct_download/source.py index ee4acfa2a..1e0da7750 100644 --- a/shelfmark/release_sources/direct_download/source.py +++ b/shelfmark/release_sources/direct_download/source.py @@ -116,6 +116,12 @@ def total_results(self) -> int | None: provider = registry.provider_by_id("annas_archive", self._providers) return getattr(provider, "total_results", None) + @property + def filtered_out_count(self) -> int: + """Returns the number of results filtered out by language.""" + provider = registry.provider_by_id("annas_archive", self._providers) + return getattr(provider, "filtered_out_count", 0) + def get_column_config(self) -> ReleaseColumnConfig: """Column configuration for Direct Download source. diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index 4433c1606..997ec6223 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -492,6 +492,7 @@ function App() { loadMore, totalFound, directTotalResults, + directFilteredOut, resultsSourceUrl, } = useSearch({ showToast, @@ -2612,6 +2613,7 @@ function App() { }} totalFound={totalFound} directTotalResults={directTotalResults} + directFilteredOut={directFilteredOut} onShowToast={showToast} resultsSourceUrl={resultsSourceUrl} /> diff --git a/src/frontend/src/components/ReleaseModal.tsx b/src/frontend/src/components/ReleaseModal.tsx index 91fe44ede..2a86cd035 100644 --- a/src/frontend/src/components/ReleaseModal.tsx +++ b/src/frontend/src/components/ReleaseModal.tsx @@ -1826,23 +1826,30 @@ const ReleaseModalSession = ({ {(() => { const searchInfo = releasesBySource[activeTab]?.search_info?.[activeTab]; const totalCount = searchInfo?.total_results; + const filteredOut = searchInfo?.filtered_out_count as number | undefined; if (totalCount === undefined || totalCount === null || totalCount === 0) return null; const isCapped = totalCount === '500+'; const totalCountNum = isCapped ? 500 : Number(totalCount); // AA-style: show page range + total if (totalCountNum === 1) { + const filterSuffix = filteredOut && filteredOut > 0 + ? `/${filteredOut} filtered` + : ''; return ( - Result 1 (1 Total) + Result 1 (1 Total{filterSuffix}) ); } const shownEnd = Math.min(totalCountNum, filteredReleases.length); const totalStr = isCapped ? '500+' : String(totalCountNum); + const filterSuffix = filteredOut && filteredOut > 0 + ? `/${filteredOut} filtered` + : ''; return ( - Results 1-{shownEnd} ({totalStr} Total) + Results 1-{shownEnd} ({totalStr} Total{filterSuffix}) ); })()} diff --git a/src/frontend/src/components/ResultsSection.tsx b/src/frontend/src/components/ResultsSection.tsx index ad27642ef..57a055974 100644 --- a/src/frontend/src/components/ResultsSection.tsx +++ b/src/frontend/src/components/ResultsSection.tsx @@ -35,6 +35,7 @@ interface ResultsSectionProps { totalFound?: number; // Direct mode: total result count from release sources directTotalResults?: number | string | null; + directFilteredOut?: number; onShowToast?: (message: string, type: 'success' | 'error' | 'info') => void; resultsSourceUrl?: string; } @@ -56,6 +57,7 @@ export const ResultsSection = ({ onLoadMore, totalFound, directTotalResults, + directFilteredOut, onShowToast, resultsSourceUrl, }: ResultsSectionProps) => { @@ -130,17 +132,23 @@ export const ResultsSection = ({ const totalCount = isCapped ? 500 : Number(count); // AA-style: show page range + total if (totalCount === 1) { + const filterSuffix = directFilteredOut && directFilteredOut > 0 + ? `/${directFilteredOut} filtered` + : ''; return ( - Result 1 (1 Total) + Result 1 (1 Total{filterSuffix}) ); } const shownEnd = Math.min(totalCount, books.length); const totalStr = isCapped ? '500+' : String(totalCount); + const filterSuffix = directFilteredOut && directFilteredOut > 0 + ? `/${directFilteredOut} filtered` + : ''; return ( - Results 1-{shownEnd} ({totalStr} Total) + Results 1-{shownEnd} ({totalStr} Total{filterSuffix}) ); })()} diff --git a/src/frontend/src/hooks/useSearch.ts b/src/frontend/src/hooks/useSearch.ts index 973a0382d..bd6441cb6 100644 --- a/src/frontend/src/hooks/useSearch.ts +++ b/src/frontend/src/hooks/useSearch.ts @@ -55,6 +55,8 @@ interface UseSearchReturn { totalFound: number; // Direct mode: total result count from release sources (e.g., "500+" for capped) directTotalResults: number | string | null; + // Direct mode: number of results filtered out by language + directFilteredOut: number; // Source URL and title for the current result set (e.g. Hardcover list page) resultsSourceUrl: string | undefined; resultsSourceTitle: string | undefined; @@ -96,6 +98,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { const [totalFound, setTotalFound] = useState(0); // Direct mode: total result count from release sources (e.g., Anna's Archive) const [directTotalResults, setDirectTotalResults] = useState(null); + const [directFilteredOut, setDirectFilteredOut] = useState(0); const [resultsSourceUrl, setResultsSourceUrl] = useState(); const [resultsSourceTitle, setResultsSourceTitle] = useState(); @@ -282,7 +285,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { setLastSearchQuery(query); try { - const { books: results, totalResults } = await searchBooks(query); + const { books: results, totalResults, filteredOutCount } = await searchBooks(query); if (results.length > 0) { // When no explicit server-side sort is selected (empty or "downloads"), @@ -293,9 +296,11 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { const sorted = isDownloadsSort ? sortBooksByDownloads(results) : results; setBooks(sorted); setDirectTotalResults(totalResults); + setDirectFilteredOut(filteredOutCount); } else { showToast('No results found', 'error'); setDirectTotalResults(null); + setDirectFilteredOut(0); } } catch (error) { if (error instanceof AuthenticationError) { @@ -346,6 +351,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { setHasMore(false); setTotalFound(0); setDirectTotalResults(null); + setDirectFilteredOut(0); setResultsSourceUrl(undefined); setResultsSourceTitle(undefined); lastSearchParamsRef.current = null; @@ -424,6 +430,7 @@ export function useSearch(options: UseSearchOptions): UseSearchReturn { loadMore, totalFound, directTotalResults, + directFilteredOut, resultsSourceUrl, resultsSourceTitle, }; diff --git a/src/frontend/src/services/api.ts b/src/frontend/src/services/api.ts index 9992ff8ca..2bd138f79 100644 --- a/src/frontend/src/services/api.ts +++ b/src/frontend/src/services/api.ts @@ -281,8 +281,8 @@ async function fetchJSON( // API functions export const searchBooks = async ( query: string, -): Promise<{ books: Book[]; totalResults: number | string | null }> => { - if (!query) return { books: [], totalResults: null }; +): Promise<{ books: Book[]; totalResults: number | string | null; filteredOutCount: number }> => { + if (!query) return { books: [], totalResults: null, filteredOutCount: 0 }; const response = await fetchJSON( `${API_BASE}/releases?source=direct_download&${query}`, {}, @@ -292,7 +292,8 @@ export const searchBooks = async ( const firstSource = response.sources_searched?.[0]; const searchInfo = firstSource ? response.search_info?.[firstSource] : undefined; const totalResults = searchInfo?.total_results ?? null; - return { books: response.releases.map(transformReleaseToDirectBook), totalResults }; + const filteredOutCount = (searchInfo?.filtered_out_count as number) ?? 0; + return { books: response.releases.map(transformReleaseToDirectBook), totalResults, filteredOutCount }; }; // Metadata search response type (internal) diff --git a/src/frontend/src/types/index.ts b/src/frontend/src/types/index.ts index 38f1a8eb5..9fbb19bad 100644 --- a/src/frontend/src/types/index.ts +++ b/src/frontend/src/types/index.ts @@ -486,6 +486,7 @@ export interface Release { export interface SourceSearchInfo { search_type: 'isbn' | 'title_author' | 'categories' | 'expanded' | 'manual' | 'query'; total_results?: number | string | null; + filtered_out_count?: number; } // Response from /api/releases endpoint diff --git a/tests/direct_download/test_search_budget.py b/tests/direct_download/test_search_budget.py index 8b37702c0..e73397b47 100644 --- a/tests/direct_download/test_search_budget.py +++ b/tests/direct_download/test_search_budget.py @@ -99,7 +99,7 @@ def fake_search_books(query, _filters): deadline = search_deadline.current() if deadline is not None: deadline.event.set() - return ([], None) + return ([], None, 0) monkeypatch.setattr(aa, "search_books", fake_search_books) monkeypatch.setattr(aa, "ensure_available", lambda: None) @@ -135,7 +135,7 @@ def fake_search_books(query, _filters): deadline = search_deadline.current() if deadline is not None: deadline.event.set() - return ([], None) + return ([], None, 0) monkeypatch.setattr(aa, "search_books", fake_search_books) monkeypatch.setattr(aa, "ensure_available", lambda: None) diff --git a/tests/direct_download/test_search_queries.py b/tests/direct_download/test_search_queries.py index a78e511ad..5603621ba 100644 --- a/tests/direct_download/test_search_queries.py +++ b/tests/direct_download/test_search_queries.py @@ -34,7 +34,7 @@ def test_uses_search_title_for_english_queries(self, monkeypatch): def fake_search_books(query: str, filters, **kwargs): del kwargs captured.append(query) - return ([], None) + return ([], None, 0) _enable_direct_download(monkeypatch) @@ -77,7 +77,7 @@ def test_deduplicates_results_across_localized_queries(self, monkeypatch): def fake_search_books(query: str, filters, **kwargs): del kwargs captured.append((query, filters.lang)) - return (records_by_query[query], None) + return (records_by_query[query], None, len(records_by_query[query])) _enable_direct_download(monkeypatch) @@ -121,8 +121,8 @@ def fake_search_books(query: str, filters, **kwargs): del kwargs captured.append((query, filters.lang)) if filters.lang: - return ([], None) - return (fallback_results[query], None) + return ([], None, 0) + return (fallback_results[query], None, len(fallback_results[query])) _enable_direct_download(monkeypatch) @@ -160,8 +160,8 @@ def fake_search_books(query: str, filters, **kwargs): del kwargs captured.append((query, filters.lang, filters.format)) if filters.lang: - return ([], None) - return ([_browse_record("manual-1", "Manual result")], None) + return ([], None, 0) + return ([_browse_record("manual-1", "Manual result")], None, 1) _enable_direct_download(monkeypatch) @@ -346,7 +346,7 @@ def _fake_html_get_page(url: str, selector, **_kwargs): monkeypatch.setattr(aa.downloader, "html_get_page", _fake_html_get_page) - records, _ = aa.search_books("demo", SearchFilters(lang=["fr"], format=["pdf"])) + records, _, _ = aa.search_books("demo", SearchFilters(lang=["fr"], format=["pdf"])) assert "&lang=" not in captured_url["url"] assert len(records) == 1 @@ -390,7 +390,7 @@ def _fake_html_get_page(url: str, selector, **_kwargs): monkeypatch.setattr(aa.downloader, "html_get_page", _fake_html_get_page) - records, _ = aa.search_books("demo", SearchFilters(lang=["en"], format=["pdf"])) + records, _, _ = aa.search_books("demo", SearchFilters(lang=["en"], format=["pdf"])) # AA already narrowed by &lang=; its free-text language cells must not be re-matched. assert "&lang=en" in captured_url["url"] From 093fb14ef0388d3e7882e6123eb2c133d411fa65 Mon Sep 17 00:00:00 2001 From: Paul Rimmer Date: Mon, 28 Sep 2026 15:32:39 +0700 Subject: [PATCH 4/5] Add support for Anna's Archive stars and if a book release has them show how many in the views --- .../direct_download/annas_archive.py | 28 +++++++++++++------ .../release_sources/direct_download/source.py | 14 ++++++++++ src/frontend/src/components/DetailsModal.tsx | 3 +- .../src/components/resultsViews/CardView.tsx | 11 +++++--- .../components/resultsViews/CompactView.tsx | 11 +++++--- .../src/components/resultsViews/ListView.tsx | 7 +++-- src/frontend/src/types/index.ts | 16 +++++++++++ src/frontend/src/utils/bookTransformers.ts | 3 ++ 8 files changed, 73 insertions(+), 20 deletions(-) diff --git a/shelfmark/release_sources/direct_download/annas_archive.py b/shelfmark/release_sources/direct_download/annas_archive.py index 2f5b0bfd7..6b93c1836 100644 --- a/shelfmark/release_sources/direct_download/annas_archive.py +++ b/shelfmark/release_sources/direct_download/annas_archive.py @@ -858,19 +858,22 @@ def search_books( return (books, total_count, raw_page_count) -def _fetch_download_count_inline(book_id: str) -> int | None: - """Fetch the download count for a single book from Anna's Archive inline_info API.""" +def _fetch_download_count_inline(book_id: str) -> tuple[int | None, int | None]: + """Fetch the download count and star rating for a single book from inline_info API. + + Returns a tuple of (downloads_total, great_quality_count), each possibly None. + """ try: url = f"{network.get_aa_base_url()}/dyn/md5/inline_info/{book_id}" resp = requests.get(url, timeout=5, headers={"Accept": "application/json"}) if resp.status_code == 200: data = resp.json() - count = data.get("downloads_total") - if count is not None: - return count + downloads = data.get("downloads_total") + stars = data.get("great_quality_count") + return (downloads, stars) except Exception: logger.debug("Failed to fetch download count for %s", book_id, exc_info=True) - return None + return (None, None) def _enrich_search_results_with_downloads(books: list[BrowseRecord]) -> None: @@ -889,14 +892,17 @@ def _enrich_search_results_with_downloads(books: list[BrowseRecord]) -> None: # Fetch counts in parallel using the inline_info API (cheaper than summary) counts: dict[str, int] = {} + stars: dict[str, int] = {} with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: futures = {executor.submit(_fetch_download_count_inline, bid): bid for bid in book_ids} for future in concurrent.futures.as_completed(futures): bid = futures[future] try: - count = future.result() - if count is not None: - counts[bid] = count + download_count, star_count = future.result() + if download_count is not None: + counts[bid] = download_count + if star_count is not None: + stars[bid] = star_count except Exception: logger.debug("Failed to fetch download count for %s", bid, exc_info=True) @@ -914,6 +920,10 @@ def _enrich_search_results_with_downloads(books: list[BrowseRecord]) -> None: if book.info is None: book.info = {} book.info["Downloads"] = [str(counts[book.id])] + if book.id in stars: + if book.info is None: + book.info = {} + book.info["Stars"] = [str(stars[book.id])] def get_book_info(book_id: str, *, fetch_download_count: bool = True) -> BrowseRecord: diff --git a/shelfmark/release_sources/direct_download/source.py b/shelfmark/release_sources/direct_download/source.py index 1e0da7750..b91747847 100644 --- a/shelfmark/release_sources/direct_download/source.py +++ b/shelfmark/release_sources/direct_download/source.py @@ -51,6 +51,19 @@ def _extract_downloads(record: BrowseRecord) -> int | None: return downloads +def _extract_stars(record: BrowseRecord) -> int | None: + """Extract star rating from record info for Release.extra.stars.""" + stars = None + if record.info and "Stars" in record.info: + stars_value = record.info["Stars"] + if isinstance(stars_value, list) and len(stars_value) > 0: + with contextlib.suppress(ValueError, TypeError): + stars = int(stars_value[0]) + elif isinstance(stars_value, (int, float)): + stars = int(stars_value) + return stars + + def _browse_record_to_release(record: BrowseRecord) -> Release: """Convert a browse record to a Release object. @@ -82,6 +95,7 @@ def _browse_record_to_release(record: BrowseRecord) -> Release: "info": record.info, "direct_download_provider": provider_id, "downloads": _extract_downloads(record), + "stars": _extract_stars(record), # Kept for older frontends and persisted request payloads. "web_provider": provider_id if provider_id != "annas_archive" else None, }, diff --git a/src/frontend/src/components/DetailsModal.tsx b/src/frontend/src/components/DetailsModal.tsx index d5d598a2f..8a8986289 100644 --- a/src/frontend/src/components/DetailsModal.tsx +++ b/src/frontend/src/components/DetailsModal.tsx @@ -104,6 +104,7 @@ export const DetailsModal = ({ // Universal mode: Year, Genres (no language, no publisher - often blank from providers) // Direct Download mode: Year, Language, Format, Size, Downloads const downloadCount = book.info?.Downloads?.[0]; + const starCount = book.info?.Stars?.[0]; const metadata = isMetadata ? [ { label: 'Year', value: book.year || '-' }, @@ -117,7 +118,7 @@ export const DetailsModal = ({ { label: 'Format', value: book.format || '-' }, { label: 'Size', value: book.size || '-' }, ...(downloadCount - ? [{ label: 'Downloads', value: Number(downloadCount).toLocaleString() }] + ? [{ label: 'Downloads', value: `${Number(downloadCount).toLocaleString()}${starCount ? ` / ⭐${starCount}` : ''}` }] : []), ]; diff --git a/src/frontend/src/components/resultsViews/CardView.tsx b/src/frontend/src/components/resultsViews/CardView.tsx index 9ecbd7b5f..0fe5b83ea 100644 --- a/src/frontend/src/components/resultsViews/CardView.tsx +++ b/src/frontend/src/components/resultsViews/CardView.tsx @@ -2,7 +2,7 @@ import { useState } from 'react'; import { useSearchMode } from '../../contexts/SearchModeContext'; import type { Book, ButtonStateInfo } from '../../types'; -import { getDownloadsCount } from '../../types'; +import { getDownloadsCount, getStarCount } from '../../types'; import { bookSupportsTargets } from '../../utils/bookTargetLoader'; import { BookActionButton } from '../BookActionButton'; import { BookTargetDropdown } from '../BookTargetDropdown'; @@ -215,12 +215,15 @@ export const CardView = ({ {searchMode !== 'universal' && (() => { const d = getDownloadsCount(book); - return d != null && d > 0 ? ( + const s = getStarCount(book); + if (d == null || d <= 0) return null; + const text = s != null && s > 0 ? `${d.toLocaleString()}/⭐${s}` : d.toLocaleString(); + return ( <> {' '} - • {d.toLocaleString()}{' '} + • {text}{' '} - ) : null; + ); })()}

)} diff --git a/src/frontend/src/components/resultsViews/CompactView.tsx b/src/frontend/src/components/resultsViews/CompactView.tsx index c52c5d2f4..e3965dae0 100644 --- a/src/frontend/src/components/resultsViews/CompactView.tsx +++ b/src/frontend/src/components/resultsViews/CompactView.tsx @@ -2,7 +2,7 @@ import { useState } from 'react'; import { useSearchMode } from '../../contexts/SearchModeContext'; import type { Book, ButtonStateInfo } from '../../types'; -import { getDownloadsCount } from '../../types'; +import { getDownloadsCount, getStarCount } from '../../types'; import { bookSupportsTargets } from '../../utils/bookTargetLoader'; import { BookActionButton } from '../BookActionButton'; import { BookTargetDropdown } from '../BookTargetDropdown'; @@ -226,12 +226,15 @@ export const CompactView = ({ )} {(() => { const d = getDownloadsCount(book); - return d != null && d > 0 ? ( + const s = getStarCount(book); + if (d == null || d <= 0) return null; + const text = s != null && s > 0 ? `${d.toLocaleString()}/⭐${s}` : d.toLocaleString(); + return ( <> {' '} - • {d.toLocaleString()}{' '} + • {text}{' '} - ) : null; + ); })()}
)} diff --git a/src/frontend/src/components/resultsViews/ListView.tsx b/src/frontend/src/components/resultsViews/ListView.tsx index 7c83e3fb5..473a72ff2 100644 --- a/src/frontend/src/components/resultsViews/ListView.tsx +++ b/src/frontend/src/components/resultsViews/ListView.tsx @@ -2,7 +2,7 @@ import { useState } from 'react'; import { useSearchMode } from '../../contexts/SearchModeContext'; import type { Book, ButtonStateInfo, DisplayField } from '../../types'; -import { getDownloadsCount } from '../../types'; +import { getDownloadsCount, getStarCount } from '../../types'; import { bookSupportsTargets } from '../../utils/bookTargetLoader'; import { getFormatColor, getLanguageColor } from '../../utils/colorMaps'; import { BookActionButton } from '../BookActionButton'; @@ -299,7 +299,10 @@ export const ListView = ({
{(() => { const d = getDownloadsCount(book); - return d != null && d > 0 ? d.toLocaleString() : '-'; + const s = getStarCount(book); + if (d == null || d <= 0) return '-'; + const text = s != null && s > 0 ? `${d.toLocaleString()}/⭐${s}` : d.toLocaleString(); + return text; })()}
)} diff --git a/src/frontend/src/types/index.ts b/src/frontend/src/types/index.ts index 9fbb19bad..e67a9c214 100644 --- a/src/frontend/src/types/index.ts +++ b/src/frontend/src/types/index.ts @@ -58,6 +58,7 @@ export interface Book { username?: string; retry_available?: boolean; downloads?: number; + stars?: number; extra?: Record; } @@ -76,6 +77,21 @@ export function getDownloadsCount(book: Book): number | null { return null; } +/** + * Extract star rating from a book's data. + * Checks both the direct `stars` field and the `extra.stars` fallback. + */ +export function getStarCount(book: Book): number | null { + if (book.stars != null && book.stars > 0) { + return book.stars; + } + const extraStars = book.extra?.stars; + if (extraStars != null && typeof extraStars === 'number' && extraStars > 0) { + return extraStars; + } + return null; +} + // Status response types export interface StatusData { queued?: Record; diff --git a/src/frontend/src/utils/bookTransformers.ts b/src/frontend/src/utils/bookTransformers.ts index 546f5e9dd..bbaa6281f 100644 --- a/src/frontend/src/utils/bookTransformers.ts +++ b/src/frontend/src/utils/bookTransformers.ts @@ -64,6 +64,7 @@ interface SourceBackedBookData { format?: unknown; size?: unknown; downloads?: unknown; + stars?: unknown; preview?: unknown; publisher?: unknown; info?: Record; @@ -161,6 +162,7 @@ const transformSourceBackedDataToBook = (data: SourceBackedBookData): Book => { format: toOptionalText(data.format), size: toOptionalText(data.size), downloads: typeof data.downloads === 'number' ? data.downloads : undefined, + stars: typeof data.stars === 'number' ? data.stars : undefined, preview: toOptionalText(data.preview), publisher: toOptionalText(data.publisher), info: data.info, @@ -186,6 +188,7 @@ export function transformReleaseToDirectBook(release: Release): Book { format: release.format, size: release.size, downloads: extra.downloads, + stars: extra.stars, preview: extra.preview, publisher: extra.publisher, info: parseBookInfo(extra.info), From f10de7d3663c3eb6a300287911a65680d741df16 Mon Sep 17 00:00:00 2001 From: Paul Rimmer Date: Mon, 28 Sep 2026 16:25:10 +0700 Subject: [PATCH 5/5] Fix linter findings and broken tests --- .../direct_download/annas_archive.py | 27 ++++++++++++------- src/frontend/src/components/DetailsModal.tsx | 7 ++++- src/frontend/src/components/ReleaseModal.tsx | 12 ++++----- .../src/components/ResultsSection.tsx | 10 +++---- .../src/components/resultsViews/CardView.tsx | 3 ++- .../components/resultsViews/CompactView.tsx | 3 ++- .../src/components/resultsViews/ListView.tsx | 3 ++- src/frontend/src/services/api.ts | 8 ++++-- tests/download/test_search_warmup.py | 4 +-- tests/download/test_warmup_yields_to_user.py | 2 +- 10 files changed, 48 insertions(+), 31 deletions(-) diff --git a/shelfmark/release_sources/direct_download/annas_archive.py b/shelfmark/release_sources/direct_download/annas_archive.py index 6b93c1836..e79ed37bd 100644 --- a/shelfmark/release_sources/direct_download/annas_archive.py +++ b/shelfmark/release_sources/direct_download/annas_archive.py @@ -887,7 +887,8 @@ def _enrich_search_results_with_downloads(books: list[BrowseRecord]) -> None: _enrich_t0 = time.perf_counter() logger.debug( - "Fetching download counts for %d books via inline_info", len(book_ids), + "Fetching download counts for %d books via inline_info", + len(book_ids), ) # Fetch counts in parallel using the inline_info API (cheaper than summary) @@ -2015,9 +2016,7 @@ def _search_books_paginated( enriched once after the last page is fetched. """ seen_ids: set[str] = set() - all_results: list[BrowseRecord] = ( - list(first_page_results) if first_page_results else [] - ) + all_results: list[BrowseRecord] = list(first_page_results) if first_page_results else [] if first_page_results: for bi in first_page_results: seen_ids.add(bi.id) @@ -2162,12 +2161,16 @@ def _search( max_pages = min(aa_page_limit, 10) # reasonable default when total is unknown results = self._search_books_paginated( - query, filters_used, total_pages=max_pages, + query, + filters_used, + total_pages=max_pages, first_page_results=results, ) _elapsed = time.perf_counter() - _search_t0 logger.info( - "AA provider search complete: %d records in %.2fs", len(results), _elapsed, + "AA provider search complete: %d records in %.2fs", + len(results), + _elapsed, ) return results @@ -2189,7 +2192,9 @@ def _search( self._total_results = isbn_total self._filtered_out_count = isbn_raw - len(results) logger.info( - "AA provider search complete: %d records in %.2fs", len(results), _elapsed, + "AA provider search complete: %d records in %.2fs", + len(results), + _elapsed, ) return results logger.debug("No ISBN results, falling back to title+author") @@ -2299,7 +2304,9 @@ def _search( for page_num in range(2, max_pages + 1): if search_deadline.expired(): break - books, _, raw_count = self._search_books(query, SearchFilters(), page=page_num) + books, _, raw_count = self._search_books( + query, SearchFilters(), page=page_num + ) for bi in books: if bi.id not in seen_ids: seen_ids.add(bi.id) @@ -2314,6 +2321,8 @@ def _search( self._filtered_out_count = filtered_out _elapsed = time.perf_counter() - _search_t0 logger.info( - "AA provider search complete: %d records in %.2fs", len(all_results), _elapsed, + "AA provider search complete: %d records in %.2fs", + len(all_results), + _elapsed, ) return all_results diff --git a/src/frontend/src/components/DetailsModal.tsx b/src/frontend/src/components/DetailsModal.tsx index 8a8986289..8d36b706b 100644 --- a/src/frontend/src/components/DetailsModal.tsx +++ b/src/frontend/src/components/DetailsModal.tsx @@ -118,7 +118,12 @@ export const DetailsModal = ({ { label: 'Format', value: book.format || '-' }, { label: 'Size', value: book.size || '-' }, ...(downloadCount - ? [{ label: 'Downloads', value: `${Number(downloadCount).toLocaleString()}${starCount ? ` / ⭐${starCount}` : ''}` }] + ? [ + { + label: 'Downloads', + value: `${Number(downloadCount).toLocaleString()}${starCount ? ` / ⭐${starCount}` : ''}`, + }, + ] : []), ]; diff --git a/src/frontend/src/components/ReleaseModal.tsx b/src/frontend/src/components/ReleaseModal.tsx index 2a86cd035..3745774bc 100644 --- a/src/frontend/src/components/ReleaseModal.tsx +++ b/src/frontend/src/components/ReleaseModal.tsx @@ -1826,16 +1826,15 @@ const ReleaseModalSession = ({ {(() => { const searchInfo = releasesBySource[activeTab]?.search_info?.[activeTab]; const totalCount = searchInfo?.total_results; - const filteredOut = searchInfo?.filtered_out_count as number | undefined; + const filteredOut = searchInfo?.filtered_out_count; if (totalCount === undefined || totalCount === null || totalCount === 0) return null; const isCapped = totalCount === '500+'; const totalCountNum = isCapped ? 500 : Number(totalCount); // AA-style: show page range + total if (totalCountNum === 1) { - const filterSuffix = filteredOut && filteredOut > 0 - ? `/${filteredOut} filtered` - : ''; + const filterSuffix = + filteredOut && filteredOut > 0 ? `/${filteredOut} filtered` : ''; return ( Result 1 (1 Total{filterSuffix}) @@ -1844,9 +1843,8 @@ const ReleaseModalSession = ({ } const shownEnd = Math.min(totalCountNum, filteredReleases.length); const totalStr = isCapped ? '500+' : String(totalCountNum); - const filterSuffix = filteredOut && filteredOut > 0 - ? `/${filteredOut} filtered` - : ''; + const filterSuffix = + filteredOut && filteredOut > 0 ? `/${filteredOut} filtered` : ''; return ( Results 1-{shownEnd} ({totalStr} Total{filterSuffix}) diff --git a/src/frontend/src/components/ResultsSection.tsx b/src/frontend/src/components/ResultsSection.tsx index 57a055974..d3db7b891 100644 --- a/src/frontend/src/components/ResultsSection.tsx +++ b/src/frontend/src/components/ResultsSection.tsx @@ -132,9 +132,8 @@ export const ResultsSection = ({ const totalCount = isCapped ? 500 : Number(count); // AA-style: show page range + total if (totalCount === 1) { - const filterSuffix = directFilteredOut && directFilteredOut > 0 - ? `/${directFilteredOut} filtered` - : ''; + const filterSuffix = + directFilteredOut && directFilteredOut > 0 ? `/${directFilteredOut} filtered` : ''; return ( Result 1 (1 Total{filterSuffix}) @@ -143,9 +142,8 @@ export const ResultsSection = ({ } const shownEnd = Math.min(totalCount, books.length); const totalStr = isCapped ? '500+' : String(totalCount); - const filterSuffix = directFilteredOut && directFilteredOut > 0 - ? `/${directFilteredOut} filtered` - : ''; + const filterSuffix = + directFilteredOut && directFilteredOut > 0 ? `/${directFilteredOut} filtered` : ''; return ( Results 1-{shownEnd} ({totalStr} Total{filterSuffix}) diff --git a/src/frontend/src/components/resultsViews/CardView.tsx b/src/frontend/src/components/resultsViews/CardView.tsx index 0fe5b83ea..a6fd161fc 100644 --- a/src/frontend/src/components/resultsViews/CardView.tsx +++ b/src/frontend/src/components/resultsViews/CardView.tsx @@ -217,7 +217,8 @@ export const CardView = ({ const d = getDownloadsCount(book); const s = getStarCount(book); if (d == null || d <= 0) return null; - const text = s != null && s > 0 ? `${d.toLocaleString()}/⭐${s}` : d.toLocaleString(); + const text = + s != null && s > 0 ? `${d.toLocaleString()}/⭐${s}` : d.toLocaleString(); return ( <> {' '} diff --git a/src/frontend/src/components/resultsViews/CompactView.tsx b/src/frontend/src/components/resultsViews/CompactView.tsx index e3965dae0..95caa6796 100644 --- a/src/frontend/src/components/resultsViews/CompactView.tsx +++ b/src/frontend/src/components/resultsViews/CompactView.tsx @@ -228,7 +228,8 @@ export const CompactView = ({ const d = getDownloadsCount(book); const s = getStarCount(book); if (d == null || d <= 0) return null; - const text = s != null && s > 0 ? `${d.toLocaleString()}/⭐${s}` : d.toLocaleString(); + const text = + s != null && s > 0 ? `${d.toLocaleString()}/⭐${s}` : d.toLocaleString(); return ( <> {' '} diff --git a/src/frontend/src/components/resultsViews/ListView.tsx b/src/frontend/src/components/resultsViews/ListView.tsx index 473a72ff2..4bdd62209 100644 --- a/src/frontend/src/components/resultsViews/ListView.tsx +++ b/src/frontend/src/components/resultsViews/ListView.tsx @@ -301,7 +301,8 @@ export const ListView = ({ const d = getDownloadsCount(book); const s = getStarCount(book); if (d == null || d <= 0) return '-'; - const text = s != null && s > 0 ? `${d.toLocaleString()}/⭐${s}` : d.toLocaleString(); + const text = + s != null && s > 0 ? `${d.toLocaleString()}/⭐${s}` : d.toLocaleString(); return text; })()}
diff --git a/src/frontend/src/services/api.ts b/src/frontend/src/services/api.ts index 2bd138f79..cfc4b5c57 100644 --- a/src/frontend/src/services/api.ts +++ b/src/frontend/src/services/api.ts @@ -292,8 +292,12 @@ export const searchBooks = async ( const firstSource = response.sources_searched?.[0]; const searchInfo = firstSource ? response.search_info?.[firstSource] : undefined; const totalResults = searchInfo?.total_results ?? null; - const filteredOutCount = (searchInfo?.filtered_out_count as number) ?? 0; - return { books: response.releases.map(transformReleaseToDirectBook), totalResults, filteredOutCount }; + const filteredOutCount = searchInfo?.filtered_out_count ?? 0; + return { + books: response.releases.map(transformReleaseToDirectBook), + totalResults, + filteredOutCount, + }; }; // Metadata search response type (internal) diff --git a/tests/download/test_search_warmup.py b/tests/download/test_search_warmup.py index 3b00e6053..51ac904cf 100644 --- a/tests/download/test_search_warmup.py +++ b/tests/download/test_search_warmup.py @@ -82,7 +82,7 @@ def test_successful_warmup_reports_true(monkeypatch, warmup): def fake_search(query, _filters): seen.append(query) - return ["a", "b"] + return ["a", "b"], None, 2 monkeypatch.setattr(dd, "search_books", fake_search) @@ -96,7 +96,7 @@ def test_empty_results_are_not_an_error(monkeypatch, warmup): import shelfmark.release_sources.direct_download as dd monkeypatch.setattr(mirrors, "has_aa_mirror_configuration", lambda: True) - monkeypatch.setattr(dd, "search_books", lambda q, f: []) + monkeypatch.setattr(dd, "search_books", lambda q, f: ([], None, 0)) assert warmup.run_warmup() is False diff --git a/tests/download/test_warmup_yields_to_user.py b/tests/download/test_warmup_yields_to_user.py index 1ab8da333..1e6cba441 100644 --- a/tests/download/test_warmup_yields_to_user.py +++ b/tests/download/test_warmup_yields_to_user.py @@ -22,7 +22,7 @@ def test_warmup_runs_when_nobody_has_searched(monkeypatch): searched: list[str] = [] monkeypatch.setattr( "shelfmark.release_sources.direct_download.search_books", - lambda query, _filters: (searched.append(query), ["a result"], None)[1:], + lambda query, _filters: (searched.append(query), ["a result"], None, 1)[1:], ) assert warmup.run_warmup() is True