From 85b0b17f43be56d1922f5ae45dc26a48dd84444a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 17:08:56 +0000 Subject: [PATCH] fix: don't grab past MAM's unsatisfied limit when its count lags; respect its download freeze MAM only counts a snatch once its tracker hears from the torrent client. While it rejected the client's announces (an unregistered IP after mousehole failed), its count stayed ~20 short and Shelfmark grabbed up to the limit, freezing downloads for a day. - Each grab also counts the MAM torrents in the torrent client still downloading or short of 72 hours' seeding, and goes by the higher of that and MAM's count. - MAM downloads wait while the client's MAM torrents show rejected announces ("Unrecognized host/PassKey"), and while MAM can't be read (the per-grab check no longer fails open). - MAM's download freeze is read from a refused torrent download, or entered by hand in the account panel; MAM downloads wait it out, and the top bar counts down to it. Kept across restarts. - Deluge, qBittorrent and Transmission report each MAM torrent's tracker error; qBittorrent's torrents with no working tracker are no longer missed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0184DMrDNkAEmGzPntj3ziqB --- CHANGELOG.md | 15 + docs/myanonamouse-enrichment.md | 8 + docs/troubleshooting.md | 9 + shelfmark/core/mam_routes.py | 23 +- shelfmark/download/clients/__init__.py | 3 + shelfmark/download/clients/deluge.py | 11 +- shelfmark/download/clients/qbittorrent.py | 63 +++- shelfmark/download/clients/torrent_utils.py | 19 + shelfmark/download/clients/transmission.py | 17 +- shelfmark/download/orchestrator.py | 71 ++-- .../release_sources/prowlarr/mam_account.py | 216 ++++++++++-- .../release_sources/prowlarr/mam_freeze.py | 129 +++++++ .../release_sources/prowlarr/settings.py | 3 +- .../src/components/MamAccountModal.tsx | 96 +++++ .../src/components/MamHeaderButton.tsx | 37 +- src/frontend/src/services/api.ts | 10 + src/frontend/src/tests/mamAccount.test.ts | 24 ++ src/frontend/src/utils/mamAccount.ts | 14 + tests/conftest.py | 18 + tests/download/test_wait_for_unsat_slot.py | 43 ++- tests/prowlarr/test_mam_account.py | 15 + tests/prowlarr/test_mam_unsat_safeguards.py | 332 ++++++++++++++++++ tests/prowlarr/test_mam_unsat_timing.py | 5 +- 23 files changed, 1093 insertions(+), 88 deletions(-) create mode 100644 shelfmark/release_sources/prowlarr/mam_freeze.py create mode 100644 tests/prowlarr/test_mam_unsat_safeguards.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c2fb3c49..d074099fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ Releases of this fork. Images: `ghcr.io/mikaeltarquin/shelfmark:` and `:latest`. +## Unreleased + +### Fixed + +- **MyAnonamouse downloads no longer run past the unsatisfied limit when MAM's count + lags.** MAM only counts a torrent once its tracker hears from the client, so while it + rejected the client's announces (an unregistered IP, say) Shelfmark went on grabbing. + Now each grab also counts the MAM torrents in the torrent client, and MAM downloads wait + while MAM rejects the client's announces or can't be read. +- **MAM's day-long download freeze is respected.** Read from a refused download (or + entered under **Download pause** in the account panel), it holds MAM downloads until it + lifts, with a countdown in the top bar. +- qBittorrent's MyAnonamouse torrents with no working tracker are no longer left out of + the slot timing. + ## v1.2.0 ### Added diff --git a/docs/myanonamouse-enrichment.md b/docs/myanonamouse-enrichment.md index 6099b0f43..ea62de7db 100644 --- a/docs/myanonamouse-enrichment.md +++ b/docs/myanonamouse-enrichment.md @@ -90,6 +90,12 @@ MyAnonamouse limits how many torrents can be unsatisfied (not yet seeded 72 hour A MAM download that's short only of unsatisfied slots doesn't have to be held back or fail. The hold-back prompt offers **Queue and wait for a slot**. A queued MAM download whose turn comes while the account is full, counting the slots kept free (or whose grab MAM refuses while at the limit), goes back to the queue as **Waiting for an unsatisfied slot on MyAnonamouse**. Waiting downloads are released, longest waiting first, once a slot is free (checked every 2 minutes), in their order on **Activity → Queued**. A torrent already in your torrent client (a retry after it was handed over) doesn't wait for a slot: it already has one. Shelfmark's own snatches that MAM's count doesn't show yet count too, since MAM's figure lags; once MAM's count has caught up they aren't counted a second time. Cancel it from Activity to drop it. Waiting downloads don't survive a restart; they show as interrupted, with Retry. +MAM's count only moves when its tracker hears from your torrent client, so Shelfmark doesn't trust it alone. Before each grab it also counts the MyAnonamouse torrents in your torrent client (Deluge, qBittorrent or Transmission) still downloading or short of 72 hours' seeding, and goes by whichever count is higher. And MAM downloads wait (saying why on **Activity → Queued**) while: + +- **MAM is rejecting the client's announces** ("Unrecognized host/PassKey", say, when the client's IP isn't registered: mousehole failing, a VPN changing address). MAM counts none of the new torrents meanwhile, so its figure is short of the truth. Downloads go again once a re-announce succeeds; force one from the client to hurry it. +- **MAM can't be read** (an expired or locked session ID, a network error). Unlike the check when you queue, this one waits rather than going ahead. +- **MAM has frozen downloads.** A grab past the unsatisfied limit gets every download refused for about a day ("Attempted to Download Past Unsatisfied limit … Time Till Download Allowed: 1d 02:04:02"). Shelfmark reads that from a refused download, or you can enter it under **Download pause** in the account panel, with **Clear** if it lifts early. The top-bar button counts down to when it lifts, in red. It's kept across restarts. + ### When slots free up The account panel (and the top-bar button and its tooltip) also estimates when unsatisfied @@ -105,3 +111,5 @@ after 72 hours minus the time it has already seeded. - Torrents still downloading haven't started their 72 hours. Torrents MAM counts but that are no longer in your client can't be timed, and don't free a slot by seeding. - Only counts and times are shown, no torrent names, so every user can see it. +- Torrents whose announces MAM rejected weren't seeding as far as MAM is concerned, so + those slots free later than shown. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index bbf11ccb9..a1fddb73a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -28,6 +28,15 @@ It's listed under **Activity → Queued → Up next**, with the countdown to the the MyAnonamouse button. See [Unsatisfied limit](myanonamouse-enrichment.md#unsatisfied-limit). +## "MyAnonamouse is rejecting …'s announces" or "MyAnonamouse has paused downloads" + +MAM downloads are on hold, whatever the free slots. **Rejected announces**: MAM doesn't +recognise the torrent client's IP or passkey, so it isn't counting new torrents; fix the +IP registration (mousehole, VPN) and the downloads go once a re-announce succeeds. +**Paused downloads**: a grab went past the unsatisfied limit and MAM refuses downloads +until the time shown. See +[Unsatisfied limit](myanonamouse-enrichment.md#unsatisfied-limit). + ## A queued book doesn't download **Activity → Queued** says what each saved pick is waiting for (unsatisfied slots, buffer, diff --git a/shelfmark/core/mam_routes.py b/shelfmark/core/mam_routes.py index 1a15f6977..07582c05d 100644 --- a/shelfmark/core/mam_routes.py +++ b/shelfmark/core/mam_routes.py @@ -14,7 +14,7 @@ from shelfmark.core.auth_modes import load_active_auth_mode from shelfmark.core.logger import setup_logger from shelfmark.core.route_guards import admin_only -from shelfmark.release_sources.prowlarr import mam_account, mam_autobuy, mam_points +from shelfmark.release_sources.prowlarr import mam_account, mam_autobuy, mam_freeze, mam_points if TYPE_CHECKING: from collections.abc import Callable @@ -80,6 +80,27 @@ def api_mam_unsat_timing() -> Response: # Counts and times only, no torrent names: every user downloads on the account. return jsonify(mam_account.unsat_timing()) + @app.route("/api/mam/freeze", methods=["POST"]) + @login_required + @admin_only + def api_mam_freeze_set() -> Response | tuple[Response, int]: + # MAM's "Time Till Download Allowed", as the site shows it ("1d 02:04:02"). + data = request.get_json(silent=True) or {} + seconds = mam_freeze.parse_duration(str(data.get("remaining") or "")) + if not seconds: + return jsonify({"error": "Enter the time left as MAM shows it, e.g. 1d 02:04:02"}), 400 + mam_freeze.set_freeze(seconds, source="manual") + logger.info("MAM download freeze entered by hand: %s s", seconds) + return jsonify(mam_account.unsat_timing()) + + @app.route("/api/mam/freeze", methods=["DELETE"]) + @login_required + @admin_only + def api_mam_freeze_clear() -> Response: + mam_freeze.clear() + logger.info("MAM download freeze cleared by hand") + return jsonify(mam_account.unsat_timing()) + @app.route("/api/mam/buffer-check", methods=["POST"]) @login_required def api_mam_buffer_check() -> Response | tuple[Response, int]: diff --git a/shelfmark/download/clients/__init__.py b/shelfmark/download/clients/__init__.py index f7714cdbb..c00dea637 100644 --- a/shelfmark/download/clients/__init__.py +++ b/shelfmark/download/clients/__init__.py @@ -182,6 +182,9 @@ class TrackerTorrent: name: str seeding_seconds: int # Time spent seeding since it completed complete: bool # Finished downloading (the seeding clock runs only after) + # The tracker's reply when its last announce failed ("Unrecognized host/PassKey"): + # MAM isn't counting or crediting the torrent meanwhile. + tracker_error: str | None = None class DownloadClient(ABC): diff --git a/shelfmark/download/clients/deluge.py b/shelfmark/download/clients/deluge.py index c85fb0df1..c1573dc81 100644 --- a/shelfmark/download/clients/deluge.py +++ b/shelfmark/download/clients/deluge.py @@ -71,6 +71,14 @@ def _get_error_message(error: object) -> tuple[str, int | None]: return str(error), None +def _tracker_error(tracker_status: object) -> str | None: + """The tracker's reason from Deluge's "Error: " status, if the announce failed.""" + text = str(tracker_status or "").strip() + if not text.lower().startswith("error"): + return None + return text.partition(":")[2].strip() or text + + def _raise_runtime_error(message: str) -> NoReturn: raise RuntimeError(message) @@ -415,7 +423,7 @@ def list_tracker_torrents(self, tracker: str) -> list[TrackerTorrent] | None: statuses = self._rpc_call( "core.get_torrents_status", {}, - ["name", "tracker_host", "seeding_time", "is_finished"], + ["name", "tracker_host", "seeding_time", "is_finished", "tracker_status"], ) if not isinstance(statuses, dict): return [] @@ -425,6 +433,7 @@ def list_tracker_torrents(self, tracker: str) -> list[TrackerTorrent] | None: name=str(status.get("name") or ""), seeding_seconds=coerce_optional_int(status.get("seeding_time")) or 0, complete=bool(status.get("is_finished")), + tracker_error=_tracker_error(status.get("tracker_status")), ) for status in statuses.values() if isinstance(status, dict) and needle in str(status.get("tracker_host") or "").lower() diff --git a/shelfmark/download/clients/qbittorrent.py b/shelfmark/download/clients/qbittorrent.py index 7038011f2..e7d1ff06d 100644 --- a/shelfmark/download/clients/qbittorrent.py +++ b/shelfmark/download/clients/qbittorrent.py @@ -44,6 +44,12 @@ _HASH_LENGTH_40 = 40 _HASH_LENGTH_ED2K = 32 _HTTP_STATUS_FORBIDDEN = HTTPStatus.FORBIDDEN +_TRACKER_NOT_WORKING = 4 # qBittorrent's tracker status +# Paused or stopped: no announces, so no tracker is "working" and none has failed either. +_IDLE_STATES = frozenset({"pausedUP", "pausedDL", "stoppedUP", "stoppedDL"}) +# (torrent hash, tracker) -> whether the torrent is on it; never changes, so asked once. +_on_tracker: dict[tuple[str, str], bool] = {} +_ON_TRACKER_MAX = 20_000 _HTTP_STATUS_NOT_FOUND = HTTPStatus.NOT_FOUND _METADATA_DOWNLOAD_STATES = {"forcedMetaDL", "metaDL"} # How long add_download waits for magnet metadata before falling back to the info @@ -296,15 +302,56 @@ def list_tracker_torrents(self, tracker: str) -> list[TrackerTorrent] | None: if error: raise RuntimeError(error) needle = tracker.lower() - return [ - TrackerTorrent( - name=str(getattr(torrent, "name", "") or ""), - seeding_seconds=coerce_optional_int(getattr(torrent, "seeding_time", 0)) or 0, - complete=(coerce_optional_float(getattr(torrent, "progress", 0)) or 0) >= 1.0, + result: list[TrackerTorrent] = [] + for torrent in torrents: + tracker_error: str | None = None + working = str(getattr(torrent, "tracker", "") or "") + if working: + if needle not in working.lower(): + continue + else: + # `tracker` is the first *working* tracker: empty when every announce + # fails, just when it matters, so ask which trackers the torrent has. + torrent_hash = str(getattr(torrent, "hash", "") or "") + key = (torrent_hash, needle) + known = _on_tracker.get(key) + if known is False: + continue + idle = str(getattr(torrent, "state", "") or "") in _IDLE_STATES + if not (known and idle): # An idle one has no announce to have failed + on_it, tracker_error = self._failing_tracker(torrent_hash, needle) + if len(_on_tracker) > _ON_TRACKER_MAX: + _on_tracker.clear() + _on_tracker[key] = on_it + if not on_it: + continue + result.append( + TrackerTorrent( + name=str(getattr(torrent, "name", "") or ""), + seeding_seconds=coerce_optional_int(getattr(torrent, "seeding_time", 0)) or 0, + complete=(coerce_optional_float(getattr(torrent, "progress", 0)) or 0) >= 1.0, + tracker_error=tracker_error, + ) ) - for torrent in torrents - if needle in str(getattr(torrent, "tracker", "") or "").lower() - ] + return result + + def _failing_tracker(self, torrent_hash: str, needle: str) -> tuple[bool, str | None]: + """Whether a torrent with no working tracker is on `needle`'s, and its message.""" + if not torrent_hash: + return False, None + self._ensure_authenticated() + response = self._client._session.get( + f"{self._base_url}/api/v2/torrents/trackers", + params={"hash": torrent_hash}, + timeout=10, + ) + response.raise_for_status() + for row in response.json(): + if isinstance(row, dict) and needle in str(row.get("url") or "").lower(): + # Status 4 is "not working"; the others are still contacting it. + failed = coerce_optional_int(row.get("status")) == _TRACKER_NOT_WORKING + return True, (str(row.get("msg") or "") or "announce failed") if failed else None + return False, None def _get_torrent_info(self, download_id: str) -> tuple[SimpleNamespace | None, str | None]: """Get one torrent by its current qBittorrent hash.""" diff --git a/shelfmark/download/clients/torrent_utils.py b/shelfmark/download/clients/torrent_utils.py index 6db3608d8..9f4a9891f 100644 --- a/shelfmark/download/clients/torrent_utils.py +++ b/shelfmark/download/clients/torrent_utils.py @@ -29,6 +29,9 @@ _BTIH_DIGEST_LENGTH = 32 _BTIH_HASH_LENGTH_40 = 40 _BTIH_HASH_LENGTH_32 = 32 +_HTTP_ERROR_MIN = 400 +# A refusal page is small; anything bigger isn't one. +_REFUSAL_MAX_BYTES = 512 * 1024 _TORRENT_FETCH_ERRORS = ( requests.exceptions.RequestException, OSError, @@ -225,6 +228,19 @@ def clear_torrent_fetch_cache() -> None: _torrent_fetch_cache.clear() +def _note_tracker_refusal(body: bytes | None) -> None: + """Show a reply that isn't a torrent to MAM's freeze detection. + + MAM refuses a grab past the unsatisfied limit with a page saying so, and then + refuses every grab for a day: held downloads must wait that out. + """ + if not isinstance(body, bytes) or not body or len(body) > _REFUSAL_MAX_BYTES: + return + from shelfmark.release_sources.prowlarr import mam_freeze + + mam_freeze.note_refusal(body.decode("utf-8", errors="ignore")) + + def _fetch_torrent_info(url: str) -> TorrentInfo: """Fetch a .torrent URL and parse out the info_hash and raw torrent data. @@ -297,6 +313,8 @@ def resolve_url(current: str, location: str) -> str: logger.debug("Following redirect to: %s...", _safe_url(redirect_url)) current_url = redirect_url + if isinstance(resp.status_code, int) and resp.status_code >= _HTTP_ERROR_MIN: + _note_tracker_refusal(resp.content) resp.raise_for_status() torrent_data = resp.content @@ -318,6 +336,7 @@ def resolve_url(current: str, location: str) -> str: logger.debug("Extracted hash from torrent file: %s", info_hash) else: logger.warning("Could not extract hash from torrent file") + _note_tracker_refusal(torrent_data) return TorrentInfo(info_hash=info_hash, torrent_data=torrent_data, is_magnet=False) except _TORRENT_FETCH_ERRORS as e: # Exception messages can repeat the source or redirect URL, including its diff --git a/shelfmark/download/clients/transmission.py b/shelfmark/download/clients/transmission.py index d9c79cdb2..0f77250ad 100644 --- a/shelfmark/download/clients/transmission.py +++ b/shelfmark/download/clients/transmission.py @@ -7,7 +7,7 @@ import importlib from contextlib import contextmanager, suppress -from typing import TYPE_CHECKING, Protocol, TypeGuard +from typing import TYPE_CHECKING, Any, Protocol, TypeGuard from shelfmark.core.config import config from shelfmark.core.logger import setup_logger @@ -139,6 +139,18 @@ def _apply_transmission_ssl_verify(client: object, url: str) -> None: logger.debug("Unable to apply Transmission TLS verify setting: %s", e) +def _tracker_error(torrent: Any, needle: str) -> str | None: + """The tracker's reason when the last announce to it failed.""" + for stats in getattr(torrent, "tracker_stats", None) or []: + if needle not in str(getattr(stats, "announce", "") or "").lower(): + continue + if getattr(stats, "has_announced", False) and not getattr( + stats, "last_announce_succeeded", True + ): + return str(getattr(stats, "last_announce_result", "") or "") or "announce failed" + return None + + @register_client("torrent") class TransmissionClient(DownloadClient): """Transmission download client using transmission-rpc library.""" @@ -367,7 +379,7 @@ def get_status(self, download_id: str) -> DownloadStatus: def list_tracker_torrents(self, tracker: str) -> list[TrackerTorrent] | None: """Every torrent on a tracker, with Transmission's seeding time.""" torrents = self._client.get_torrents( - arguments=["name", "secondsSeeding", "percentDone", "trackers"] + arguments=["name", "secondsSeeding", "percentDone", "trackers", "trackerStats"] ) needle = tracker.lower() result: list[TrackerTorrent] = [] @@ -380,6 +392,7 @@ def list_tracker_torrents(self, tracker: str) -> list[TrackerTorrent] | None: name=str(torrent.name or ""), seeding_seconds=int(torrent.seconds_seeding or 0), complete=float(torrent.percent_done or 0) >= 1.0, + tracker_error=_tracker_error(torrent, needle), ) ) return result diff --git a/shelfmark/download/orchestrator.py b/shelfmark/download/orchestrator.py index 26c2a0b91..39a036101 100644 --- a/shelfmark/download/orchestrator.py +++ b/shelfmark/download/orchestrator.py @@ -1109,10 +1109,12 @@ def _finalize_download_failure(task_id: str) -> None: # --- Waiting for a MyAnonamouse unsatisfied slot ------------------------------------- # -# MAM refuses a torrent while the account is at its unsatisfied limit. Rather than fail -# such a download (and leave it to be retried by hand), it goes back to the queue, -# marked as waiting, and is released oldest first once a slot is free: checked before -# each MAM grab, and after a grab fails while the account is at the limit. +# MAM refuses a torrent while the account is at its unsatisfied limit, and a refused +# grab freezes the account's downloads for a day. Rather than fail such a download (and +# leave it to be retried by hand), it goes back to the queue, marked as waiting, and is +# released oldest first once a slot is free: checked before each MAM grab, and after a +# grab fails while the account is at the limit. It also waits while MAM has frozen +# downloads, can't be read, or is refusing the torrent client's announces. _UNSAT_RECHECK_SECONDS = 2 * 60 UNSAT_WAIT_MESSAGE = ( @@ -1124,12 +1126,6 @@ def _finalize_download_failure(task_id: str) -> None: _slot_watcher: threading.Thread | None = None -def _mam_free_slots(task_id: str, *, refresh: bool = False) -> int | None: - from shelfmark.release_sources.prowlarr import mam_account - - return mam_account.unsat_free_slots(exclude={task_id}, refresh=refresh) - - def _torrent_in_client(task_id: str) -> bool: """Whether the task's torrent is in the torrent client already. @@ -1165,21 +1161,27 @@ def _torrent_in_client(task_id: str) -> bool: return False -def _needs_slot(task_id: str, *, refresh: bool = False) -> bool: - """Whether a MAM download has to wait: no slot is free and MAM doesn't have it yet.""" - free = _mam_free_slots(task_id, refresh=refresh) - return free is not None and free < 1 and not _torrent_in_client(task_id) +def _slot_wait(task_id: str, *, refresh: bool = False) -> str | None: + """Why a MAM download has to wait, or None: it can go, or MAM has it already.""" + from shelfmark.release_sources.prowlarr import mam_account + + allowance = mam_account.grab_allowance(exclude={task_id}, refresh=refresh) + if allowance.hold is None and (allowance.free is None or allowance.free >= 1): + return None + if _torrent_in_client(task_id): + return None + return allowance.hold or UNSAT_WAIT_MESSAGE -def _hold_for_slot(task_id: str) -> bool: +def _hold_for_slot(task_id: str, message: str = UNSAT_WAIT_MESSAGE) -> bool: """Send a picked-up MAM download back to wait for an unsatisfied slot.""" - if not book_queue.hold(task_id, UNSAT_WAIT_MESSAGE): + if not book_queue.hold(task_id, message): return False book_queue.forget_handoff(task_id) # MAM never had it with _held_lock: _held_for_slot.setdefault(task_id, time.time()) _ensure_slot_watcher() - logger.info("Download %s waits for a MyAnonamouse unsatisfied slot", task_id) + logger.info("Download %s waits for MyAnonamouse: %s", task_id, message) if ws_manager: ws_manager.broadcast_status_update(queue_status()) return True @@ -1203,6 +1205,18 @@ def order(item: tuple[str, float]) -> tuple[int, float, float]: return [task_id for task_id, _ in sorted(waiting, key=order)] +def _say_why_waiting(task_ids: list[str], message: str) -> None: + """Show why the waiting downloads still wait, when the reason has changed.""" + changed = False + for task_id in task_ids: + task = book_queue.get_task(task_id) + if task is not None and task.status_message != message: + book_queue.update_status_message(task_id, message) + changed = True + if changed and ws_manager: + ws_manager.broadcast_status_update(queue_status()) + + def release_held_downloads() -> list[str]: """Queue the waiting downloads that now fit, longest waiting first.""" from shelfmark.release_sources.prowlarr import mam_account @@ -1222,9 +1236,14 @@ def release_held_downloads() -> list[str]: if not live: return [] if mam_account.unsat_check_enabled(): - free = mam_account.unsat_free_slots(exclude=set(live), refresh=True) - if free is None: - return [] # MAM can't be read: keep waiting rather than be refused again + allowance = mam_account.grab_allowance(exclude=set(live), refresh=True) + if allowance.hold is not None: + _say_why_waiting(live, allowance.hold) + return [] + # No limit reported: MAM can't be judged, so keep waiting rather than be refused + free = allowance.free if allowance.free is not None else 0 + if free < 1: + _say_why_waiting(live, UNSAT_WAIT_MESSAGE) else: free = len(live) # The check was turned off: let them all go released: list[str] = [] @@ -1270,8 +1289,10 @@ def _process_single_download(task_id: str, cancel_flag: Event) -> None: is_mam = bool(task and task.mam_torrent_id) with _held_lock: _held_for_slot.pop(task_id, None) # Picked up: released, or retried by hand - if is_mam and not cancel_flag.is_set() and _needs_slot(task_id) and _hold_for_slot(task_id): - return + if is_mam and not cancel_flag.is_set(): + wait = _slot_wait(task_id) + if wait is not None and _hold_for_slot(task_id, wait): + return # Status will be updated through callbacks during download process # (resolving -> downloading -> complete) @@ -1291,8 +1312,10 @@ def _process_single_download(task_id: str, cancel_flag: Event) -> None: book_queue.update_download_path(task_id, download_path) book_queue.update_status(task_id, QueueStatus.COMPLETE) else: - # A refused grab at the unsatisfied limit waits for a slot instead of failing. - if is_mam and _needs_slot(task_id, refresh=True) and _hold_for_slot(task_id): + # A refused grab at the unsatisfied limit (or in a freeze it set off) waits + # instead of failing. + wait = _slot_wait(task_id, refresh=True) if is_mam else None + if wait is not None and _hold_for_slot(task_id, wait): return _finalize_download_failure(task_id) diff --git a/shelfmark/release_sources/prowlarr/mam_account.py b/shelfmark/release_sources/prowlarr/mam_account.py index 8c57bd54b..1200dd0fd 100644 --- a/shelfmark/release_sources/prowlarr/mam_account.py +++ b/shelfmark/release_sources/prowlarr/mam_account.py @@ -415,9 +415,28 @@ def _torrent_client_status() -> dict[str, Any]: UNSAT_SEED_SECONDS = 72 * 3600 # "Slots freeing up soon" looks this far ahead. UNSAT_WINDOW_SECONDS = 6 * 3600 -_TIMING_TTL_SECONDS = 60 -_timing_lock = threading.Lock() -_timing_cache: tuple[float, dict[str, Any]] | None = None +_CLIENT_TTL_SECONDS = 60 +# Snatches handed off this long before the client was read may be missing from it. +_CLIENT_LAG_SECONDS = 30 +# Tracker replies that mean MAM is refusing the client itself (its IP or passkey), not +# one torrent: "Unrecognized host/PassKey". MAM counts none of its snatches meanwhile. +_ANNOUNCE_REJECTED_RE = re.compile( + r"passkey|unrecognized\s+host|not\s+authori[sz]ed|unauthori[sz]ed", re.IGNORECASE +) +_client_lock = threading.Lock() + + +@dataclass(frozen=True) +class _ClientRead: + """The torrent client's MAM torrents, as read at `read_at`.""" + + read_at: float + name: str | None + torrents: list[Any] | None # None when it couldn't be read; `reason` says why + reason: str | None = None + + +_client_cache: _ClientRead | None = None def summarize_unsat_timing(torrents: list[Any], client_name: str | None) -> dict[str, Any]: @@ -442,36 +461,39 @@ def summarize_unsat_timing(torrents: list[Any], client_name: str | None) -> dict } -def _read_unsat_timing() -> dict[str, Any]: +def _read_client() -> _ClientRead: from shelfmark.download.clients import get_client + now = time.time() try: client = get_client("torrent") - except (ImportError, RuntimeError, ValueError) as exc: - return {"available": False, "reason": str(exc)} + except Exception as exc: # noqa: BLE001 - each client library raises its own errors + return _ClientRead(now, None, None, str(exc)) if client is None: - return {"available": False, "reason": "No torrent client is configured"} + return _ClientRead(now, None, None, "No torrent client is configured") raw_name = str(getattr(client, "name", "") or "") name = _CLIENT_LABELS.get(raw_name, raw_name.title() or None) try: torrents = client.list_tracker_torrents("myanonamouse") - except ( - OSError, - RuntimeError, - ValueError, - KeyError, - TypeError, - requests.exceptions.RequestException, - ) as exc: + except Exception as exc: # noqa: BLE001 - read before every MAM grab: never break one logger.warning("Could not read MAM torrents from %s: %s", name, exc) - return {"available": False, "client": name, "reason": f"Couldn't read {name}: {exc}"} + return _ClientRead(now, name, None, f"Couldn't read {name}: {exc}") if torrents is None: - return { - "available": False, - "client": name, - "reason": f"{name} doesn't report seeding time", - } - return summarize_unsat_timing(torrents, name) + return _ClientRead(now, name, None, f"{name} doesn't report seeding time") + return _ClientRead(now, name, list(torrents)) + + +def _client_torrents(*, refresh: bool = False) -> _ClientRead: + """The client's MAM torrents, read at most once a minute unless `refresh` is set.""" + global _client_cache + with _client_lock: + cached = _client_cache + if not refresh and cached and time.time() - cached.read_at < _CLIENT_TTL_SECONDS: + return cached + read = _read_client() + with _client_lock: + _client_cache = read + return read def _aged_timing(timing: dict[str, Any], age: float) -> dict[str, Any]: @@ -479,28 +501,87 @@ def _aged_timing(timing: dict[str, Any], age: float) -> dict[str, Any]: next_seconds = timing.get("next_seconds") if not isinstance(next_seconds, (int, float)): return timing - return {**timing, "next_seconds": max(0, int(next_seconds - age))} + # Rounded up, so a slot is never promised early. + return {**timing, "next_seconds": max(0, math.ceil(next_seconds - age))} + + +def client_unsat_count( + *, exclude: set[str] | frozenset[str] = frozenset(), refresh: bool = False +) -> int | None: + """The unsatisfied torrents MAM will count once it sees what's in the client. + + Every MAM torrent there still downloading or short of 72 hours' seeding, plus + Shelfmark's snatches handed off since (or just before) the client was read, but + not those in `exclude`. MAM's own count only moves when the tracker hears from + the client, so it misses every snatch whose announces are refused; this doesn't. + None when the client can't say. + """ + from shelfmark.core.queue import book_queue + + read = _client_torrents(refresh=refresh) + if read.torrents is None: + return None + unsat = sum( + 1 for t in read.torrents if not t.complete or t.seeding_seconds < UNSAT_SEED_SECONDS + ) + later = [ + task + for _at, task in book_queue.handoff_times_since(read.read_at - _CLIENT_LAG_SECONDS) + if task.mam_torrent_id and task.task_id not in exclude + ] + return unsat + len(later) + + +def announce_problem(*, refresh: bool = False) -> str | None: + """Why MAM isn't hearing from the torrent client, if it's refusing its announces. + + While it is, MAM counts none of the new snatches (and credits no seeding), so its + unsatisfied count can't be trusted: downloads then wait. + """ + read = _client_torrents(refresh=refresh) + rejected = [ + t.tracker_error + for t in read.torrents or [] + if getattr(t, "tracker_error", None) and _ANNOUNCE_REJECTED_RE.search(t.tracker_error) + ] + if not rejected: + return None + count = len(rejected) + return ( + f"MyAnonamouse is rejecting {read.name or 'the torrent client'}'s announces " + f'({count} torrent{"" if count == 1 else "s"}: "{rejected[0]}"). Check the ' + "client's IP is registered with MAM (mousehole, say); downloads wait until a " + "re-announce succeeds" + ) def unsat_timing(*, refresh: bool = False) -> dict[str, Any]: """When the next unsatisfied MAM torrent frees a slot, from the torrent client. An estimate: MAM keeps its own clock, so time spent paused or offline pushes the - real moment later. Cached for a minute. + real moment later. Read at most once a minute. Also says when MAM has frozen + downloads, and when it's refusing the client's announces: either one comes first. """ - global _timing_cache + from shelfmark.release_sources.prowlarr import mam_freeze + if not is_configured(): return {"available": False, "reason": "No MAM session ID is set"} - with _timing_lock: - cached = _timing_cache - if not refresh and cached: - age = time.time() - cached[0] - if age < _TIMING_TTL_SECONDS: - return _aged_timing(cached[1], age) - timing = _read_unsat_timing() - with _timing_lock: - _timing_cache = (time.time(), timing) - return timing + read = _client_torrents(refresh=refresh) + if read.torrents is None: + timing: dict[str, Any] = {"available": False, "reason": read.reason} + if read.name: + timing["client"] = read.name + else: + timing = _aged_timing( + summarize_unsat_timing(read.torrents, read.name), time.time() - read.read_at + ) + freeze = mam_freeze.current() + return { + **timing, + "frozen_seconds": mam_freeze.remaining_seconds(), + "frozen_source": freeze["source"] if freeze else None, + "announce_problem": announce_problem(), + } def connection_status() -> dict[str, Any]: @@ -586,7 +667,13 @@ def pending_unsat_count( for at, task in book_queue.handoff_times_since(since) if task.mam_torrent_id and task.task_id not in counted ] - return len(counted) + _uncounted_snatches(snatched, unsat_count) + pending = len(counted) + _uncounted_snatches(snatched, unsat_count) + if unsat_count is not None: + # The client can show more than MAM counts yet (see `client_unsat_count`). + in_client = client_unsat_count(exclude=counted) + if in_client is not None: + pending = max(pending, len(counted) + in_client - unsat_count) + return pending def unsat_free_slots( @@ -596,7 +683,8 @@ def unsat_free_slots( For a queued download about to be grabbed (or waiting for room). Counts what MAM reports plus Shelfmark's torrents snatched since shortly before the stats were read - (see `pending_unsat_count`), leaving out the tasks in `exclude` (the ones asking). + (see `pending_unsat_count`), or what's in the torrent client if that's more (see + `client_unsat_count`), leaving out the tasks in `exclude` (the ones asking). None when the check is off, MAM isn't set up or can't be read, or reports no limit. """ from shelfmark.core.queue import book_queue @@ -616,8 +704,58 @@ def unsat_free_slots( for at, task in book_queue.handoff_times_since(since) if task.mam_torrent_id and task.task_id not in exclude ] - uncounted = _uncounted_snatches(snatched, stats.unsat_count) - return stats.unsat_limit - unsat_reserve_slots() - stats.unsat_count - uncounted + used = stats.unsat_count + _uncounted_snatches(snatched, stats.unsat_count) + in_client = client_unsat_count(exclude=exclude, refresh=refresh) + if in_client is not None and in_client > used: + logger.debug("The torrent client has %d unsatisfied, MAM shows %d", in_client, used) + used = in_client + return stats.unsat_limit - unsat_reserve_slots() - used + + +@dataclass(frozen=True) +class GrabAllowance: + """How many MAM torrents may be grabbed now. + + `free` is None when there's no limit to keep to (the check is off, or MAM reports + none). `hold` says why none may be, when it's more than the slots being full. + """ + + free: int | None + hold: str | None = None + + +def grab_allowance( + *, exclude: set[str] | frozenset[str] = frozenset(), refresh: bool = False +) -> GrabAllowance: + """Whether MAM torrents may be grabbed now, and how many. + + Unlike the buffer check before a download is queued, this errs towards waiting: + a grab MAM refuses at the limit freezes the account's downloads for a day. So + nothing goes while MAM has frozen downloads, while it can't be read, or while + it's refusing the client's announces (its count is short of the truth then). + """ + from shelfmark.release_sources.prowlarr import mam_freeze + + if not unsat_check_enabled(): + return GrabAllowance(None) + frozen = mam_freeze.remaining_seconds() + if frozen is not None: + hours, minutes = frozen // 3600, frozen % 3600 // 60 + return GrabAllowance( + 0, + "MyAnonamouse has paused downloads on the account (a grab went past the " + f"unsatisfied limit): waiting {hours} h {minutes:02d} m until it lifts", + ) + try: + get_stats(refresh=refresh) + except MAM_ERRORS as exc: + return GrabAllowance( + 0, f"Waiting: couldn't read the MyAnonamouse account ({describe_error(exc)})" + ) + problem = announce_problem(refresh=refresh) + if problem: + return GrabAllowance(0, f"Waiting: {problem}") + return GrabAllowance(unsat_free_slots(exclude=exclude)) def recommended_purchase_gb(missing_bytes: int) -> int: diff --git a/shelfmark/release_sources/prowlarr/mam_freeze.py b/shelfmark/release_sources/prowlarr/mam_freeze.py new file mode 100644 index 000000000..6bc5860c1 --- /dev/null +++ b/shelfmark/release_sources/prowlarr/mam_freeze.py @@ -0,0 +1,129 @@ +"""MyAnonamouse's download freeze: after a grab past the unsatisfied limit, MAM refuses +every download for about a day ("Attempted to Download Past Unsatisfied limit ... Time +Till Download Allowed: 1d 02:04:02"). + +Read from a refused torrent download when MAM's reply says so, or entered by hand from +the site. Kept in a file so a restart doesn't forget it: grabbing during the freeze only +gets refused again. +""" + +from __future__ import annotations + +import json +import re +import threading +import time +from typing import Any + +from shelfmark.config.env import CONFIG_DIR +from shelfmark.core.logger import setup_logger + +logger = setup_logger(__name__) + +_FREEZE_FILE = "mam_freeze.json" +# When MAM says it's frozen but not for how long: its freeze is a day. +DEFAULT_FREEZE_SECONDS = 24 * 3600 +# Longer than any freeze MAM hands out; guards against a misread. +_MAX_FREEZE_SECONDS = 7 * 24 * 3600 + +_LIMIT_RE = re.compile(r"past\s+unsatisfied\s+limit", re.IGNORECASE) +# "1d 02:04:02", "2 days 1:00:00" or "23:59:59", after the label. The label and value +# can be split by markup (HTML cells, or escaped HTML in a proxy's JSON error). +_DURATION = r"(?:(\d+)\s*d\w*\s*)?(\d{1,2}):(\d{2}):(\d{2})" +_REMAINING_RE = re.compile( + r"Time\s+Till\s+Download\s+Allowed.{0,80}?" + _DURATION, re.IGNORECASE | re.DOTALL +) +_DURATION_RE = re.compile(r"^\s*" + _DURATION + r"\s*$", re.IGNORECASE) +_TAG_RE = re.compile(r"<[^>]*>") + +_lock = threading.Lock() + + +def _path() -> Any: + return CONFIG_DIR / _FREEZE_FILE + + +def _seconds(match: re.Match[str]) -> int: + days, hours, minutes, seconds = (int(g) if g else 0 for g in match.groups()) + return ((days * 24 + hours) * 60 + minutes) * 60 + seconds + + +def parse_duration(text: str) -> int | None: + """Seconds in a remaining time as MAM shows it ("1d 02:04:02"), or None.""" + match = _DURATION_RE.match(text or "") + return _seconds(match) if match else None + + +def parse_refusal(text: str) -> int | None: + """How long MAM's download freeze lasts, from a refused download's reply. + + None when the reply isn't a freeze; a day when it is but gives no time. + """ + if not text: + return None + plain = _TAG_RE.sub(" ", text) + match = _REMAINING_RE.search(plain) + if match: + return _seconds(match) + return DEFAULT_FREEZE_SECONDS if _LIMIT_RE.search(plain) else None + + +def _load() -> dict[str, Any] | None: + with _lock: + try: + data = json.loads(_path().read_text(encoding="utf-8")) + except OSError, ValueError: + return None + if not isinstance(data, dict) or not isinstance(data.get("until"), (int, float)): + return None + return data + + +def current() -> dict[str, Any] | None: + """The freeze in force now: {"until", "source"}, or None.""" + data = _load() + if data is None or data["until"] <= time.time(): + return None + return {"until": float(data["until"]), "source": str(data.get("source") or "detected")} + + +def remaining_seconds() -> int | None: + """Seconds until MAM allows downloads again, or None when it isn't frozen.""" + freeze = current() + return max(0, int(freeze["until"] - time.time())) if freeze else None + + +def set_freeze(seconds: float, *, source: str) -> float: + """Record a freeze lasting `seconds` from now; returns when it ends.""" + seconds = min(max(float(seconds), 0.0), _MAX_FREEZE_SECONDS) + until = time.time() + seconds + with _lock: + try: + _path().write_text(json.dumps({"until": until, "source": source}), encoding="utf-8") + except OSError as exc: + logger.warning("Could not save the MAM download freeze: %s", exc) + return until + + +def clear() -> None: + """Forget the freeze (MAM lifted it, or it was entered by mistake).""" + with _lock: + try: + _path().unlink(missing_ok=True) + except OSError as exc: + logger.warning("Could not clear the MAM download freeze: %s", exc) + + +def note_refusal(text: str) -> bool: + """Record a freeze if a refused download's reply is MAM's; whether it was.""" + seconds = parse_refusal(text) + if seconds is None: + return False + set_freeze(seconds, source="detected") + logger.warning( + "MyAnonamouse has frozen downloads for the unsatisfied limit: holding MAM " + "downloads for %d h %02d m", + seconds // 3600, + seconds % 3600 // 60, + ) + return True diff --git a/shelfmark/release_sources/prowlarr/settings.py b/shelfmark/release_sources/prowlarr/settings.py index 701c0998e..6fcdecf18 100644 --- a/shelfmark/release_sources/prowlarr/settings.py +++ b/shelfmark/release_sources/prowlarr/settings.py @@ -311,7 +311,8 @@ def prowlarr_config_settings() -> list[SettingsField]: "hours) at once, by user class. Before queueing MAM torrents (freeleech too), " "check that they, plus Shelfmark's MAM downloads not yet started, leave the " "slots below free. Buying upload credit doesn't help here; wait for torrents " - "to finish seeding." + "to finish seeding. Queued MAM downloads also wait while MAM can't be read, " + "rejects the torrent client's announces, or has frozen downloads." ), default=True, show_when={"field": "PROWLARR_ENABLED", "value": True}, diff --git a/src/frontend/src/components/MamAccountModal.tsx b/src/frontend/src/components/MamAccountModal.tsx index 9e2868729..8bbac9316 100644 --- a/src/frontend/src/components/MamAccountModal.tsx +++ b/src/frontend/src/components/MamAccountModal.tsx @@ -5,21 +5,25 @@ import { useEscapeKey } from '../hooks/useEscapeKey'; import { useMountEffect } from '../hooks/useMountEffect'; import { buyMamUploadCredit, + clearMamFreeze, getMamAccount, getMamAutobuy, getMamStatus, getMamUnsatTiming, runMamAutobuy, + setMamFreeze, } from '../services/api'; import { customAmountError, describeAutobuy, + describeMamHold, describeUnsatTiming, describeCheck, purchaseReasonLabel, DEFAULT_POINTS_PER_GB, DEFAULT_STEP_GB, formatGib, + formatHoursMinutes, formatPoints, formatRatio, formatUnsat, @@ -45,6 +49,91 @@ interface MamAccountModalProps { type Choice = number | 'max' | 'custom'; +/** MAM's download freeze: shown when in force, or entered by hand from the site. */ +const FreezeSection = ({ + timing, + onChange, +}: { + timing: MamUnsatTiming | null; + onChange: (timing: MamUnsatTiming) => void; +}) => { + const [remaining, setRemaining] = useState(''); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const frozen = timing?.frozen_seconds ?? null; + + const run = async (request: () => Promise) => { + setBusy(true); + setError(null); + try { + onChange(await request()); + setRemaining(''); + } catch (failure) { + setError(errorText(failure)); + } finally { + setBusy(false); + } + }; + + return ( +
+

Download pause

+ {frozen ? ( +
+

+ MyAnonamouse has paused downloads: allowed again in {formatHoursMinutes(frozen)}. MAM + downloads wait until then + {timing?.frozen_source === 'manual' ? ' (entered by hand)' : ''}. +

+ +
+ ) : ( + <> +

+ If MAM shows “Time Till Download Allowed”, enter it so MAM downloads wait it + out. Shelfmark also reads it from a refused download. +

+
{ + event.preventDefault(); + void run(() => setMamFreeze(remaining.trim())); + }} + > + setRemaining(event.target.value)} + className="w-32 rounded-full border border-(--border-muted) bg-(--bg-soft) px-3 py-1.5 text-sm" + /> + +
+ + )} + {error && ( +

+ {error} +

+ )} +
+ ); +}; + const errorText = (error: unknown): string => error instanceof Error ? error.message : 'Request failed'; @@ -365,6 +454,11 @@ export const MamAccountModal = ({ onClose }: MamAccountModalProps) => { + {timing?.announce_problem && !timing.frozen_seconds && ( +

+ {describeMamHold(timing)} +

+ )} {timing && (

Unsatisfied slots: @@ -383,6 +477,8 @@ export const MamAccountModal = ({ onClose }: MamAccountModalProps) => { )} + +

Connections

    diff --git a/src/frontend/src/components/MamHeaderButton.tsx b/src/frontend/src/components/MamHeaderButton.tsx index 062a7a593..dca97cd2f 100644 --- a/src/frontend/src/components/MamHeaderButton.tsx +++ b/src/frontend/src/components/MamHeaderButton.tsx @@ -3,6 +3,7 @@ import { useState } from 'react'; import { useDependencyEffect } from '../hooks/useMountEffect'; import { getMamRatio, getMamUnsatTiming } from '../services/api'; import { + describeMamHold, describeUnsatTiming, formatCountdown, formatRatio, @@ -18,7 +19,15 @@ import { MouseIcon } from './MouseIcon'; const REFRESH_MS = 5 * 60 * 1000; /** Ticks once a second toward `target` (ms since epoch), then calls `onDone` once. */ -const SlotCountdown = ({ target, onDone }: { target: number; onDone: () => void }) => { +const SlotCountdown = ({ + target, + onDone, + tone, +}: { + target: number; + onDone: () => void; + tone?: string; +}) => { const [now, setNow] = useState(() => Date.now()); useDependencyEffect(() => { const timer = window.setInterval(() => { @@ -32,7 +41,9 @@ const SlotCountdown = ({ target, onDone }: { target: number; onDone: () => void return () => window.clearInterval(timer); }, [target]); return ( - + { const [timing, setTiming] = useState(null); // When the next unsatisfied torrent frees a slot, on this browser's clock. const [slotAt, setSlotAt] = useState(null); + // When MAM lifts its download freeze, on this browser's clock. + const [thawAt, setThawAt] = useState(null); const [reloadKey, setReloadKey] = useState(0); useDependencyEffect(() => { @@ -81,11 +94,14 @@ export const MamHeaderButton = ({ onClick }: MamHeaderButtonProps) => { setSlotAt( typeof seconds === 'number' && seconds > 0 ? Date.now() + seconds * 1000 : null, ); + const frozen = next.frozen_seconds; + setThawAt(typeof frozen === 'number' && frozen > 0 ? Date.now() + frozen * 1000 : null); }) .catch(() => { if (cancelled) return; setTiming(null); setSlotAt(null); + setThawAt(null); }); }; load(); @@ -100,9 +116,12 @@ export const MamHeaderButton = ({ onClick }: MamHeaderButtonProps) => { const unsatCount = stats?.available ? (stats.unsat_count ?? null) : null; const unsatLimit = stats?.available ? (stats.unsat_limit ?? null) : null; const hasStats = Boolean(stats?.available); - // Out of slots: count down to the next one, if a seeding torrent will free it. - const countdownTo = hasStats && unsatSlotsFull(stats) ? slotAt : null; - const slots = describeUnsatTiming(timing); + // MAM has frozen downloads: count down to when it lifts. Otherwise, out of slots: + // count down to the next one, if a seeding torrent will free it. + const countdownTo = thawAt ?? (hasStats && unsatSlotsFull(stats) ? slotAt : null); + const hold = describeMamHold(timing); + const holdTone = hold ? 'text-red-600 dark:text-red-400' : undefined; + const slots = hold ?? describeUnsatTiming(timing); const summary = hasStats ? `Ratio ${formatRatio(ratio)}, unsatisfied ${formatUnsat(unsatCount, unsatLimit)}${ slots ? `. ${slots}` : '' @@ -127,12 +146,16 @@ export const MamHeaderButton = ({ onClick }: MamHeaderButtonProps) => { {formatRatio(ratio)} · Unsat - + {formatUnsat(unsatCount, unsatLimit)} {countdownTo !== null && ( - setReloadKey((key) => key + 1)} /> + setReloadKey((key) => key + 1)} + /> )} ) : ( diff --git a/src/frontend/src/services/api.ts b/src/frontend/src/services/api.ts index 40c89c121..2784378ba 100644 --- a/src/frontend/src/services/api.ts +++ b/src/frontend/src/services/api.ts @@ -795,6 +795,16 @@ export const deleteSavedItem = async (id: number): Promise => { export const getMamUnsatTiming = async (): Promise => fetchJSON(`${API_BASE}/mam/unsat-timing`); +/** Hold MAM downloads for `remaining`, MAM's "Time Till Download Allowed" ("1d 02:04:02"). */ +export const setMamFreeze = async (remaining: string): Promise => + fetchJSON(`${API_BASE}/mam/freeze`, { + method: 'POST', + body: JSON.stringify({ remaining }), + }); + +export const clearMamFreeze = async (): Promise => + fetchJSON(`${API_BASE}/mam/freeze`, { method: 'DELETE' }); + export const checkMamBuffer = async (releases: DownloadReleasePayload[]): Promise => fetchJSON(`${API_BASE}/mam/buffer-check`, { method: 'POST', diff --git a/src/frontend/src/tests/mamAccount.test.ts b/src/frontend/src/tests/mamAccount.test.ts index c69b0700e..e4b914cd4 100644 --- a/src/frontend/src/tests/mamAccount.test.ts +++ b/src/frontend/src/tests/mamAccount.test.ts @@ -9,6 +9,7 @@ import { formatRatio, maxAffordableGb, uploadCreditCost, + describeMamHold, describeUnsatTiming, formatCountdown, formatHoursMinutes, @@ -182,3 +183,26 @@ describe('formatCountdown', () => { expect(formatCountdown(-5)).toBe('0:00'); }); }); + +describe('describeMamHold', () => { + it('counts down a freeze, ahead of anything else', () => { + expect( + describeMamHold({ + available: true, + frozen_seconds: 26 * 3600 + 4 * 60, + announce_problem: 'MyAnonamouse is rejecting announces', + }), + ).toBe('MyAnonamouse has paused downloads (unsatisfied limit) · allowed again in 26:04'); + }); + + it('says when announces are rejected, even without a client timing', () => { + expect( + describeMamHold({ available: false, announce_problem: 'MyAnonamouse is rejecting' }), + ).toBe('MyAnonamouse is rejecting'); + }); + + it('is null when nothing holds downloads', () => { + expect(describeMamHold({ available: true, frozen_seconds: null })).toBeNull(); + expect(describeMamHold(null)).toBeNull(); + }); +}); diff --git a/src/frontend/src/utils/mamAccount.ts b/src/frontend/src/utils/mamAccount.ts index 33a312d5b..7dd9b68e8 100644 --- a/src/frontend/src/utils/mamAccount.ts +++ b/src/frontend/src/utils/mamAccount.ts @@ -165,6 +165,10 @@ export interface MamUnsatTiming { window_hours?: number; seeding?: number; // MAM torrents seeding but not yet at 72 hours downloading?: number; // Still downloading: their 72 hours haven't started + // Until MAM allows downloads again, after a grab past the unsatisfied limit froze them + frozen_seconds?: number | null; + frozen_source?: 'detected' | 'manual' | null; // Read from MAM's refusal, or entered by hand + announce_problem?: string | null; // MAM is rejecting the client's announces } /** Seconds as h:mm ("2:05"), rounded up so a slot is never promised early. */ @@ -196,6 +200,16 @@ export const describeUnsatTiming = (timing: MamUnsatTiming | null): string | nul return `Next slot in ${formatHoursMinutes(timing.next_seconds)} · ${slots}`; }; +/** Why MAM downloads are on hold whatever the free slots, or null. */ +export const describeMamHold = (timing: MamUnsatTiming | null): string | null => { + if (timing?.frozen_seconds) { + return `MyAnonamouse has paused downloads (unsatisfied limit) · allowed again in ${formatHoursMinutes( + timing.frozen_seconds, + )}`; + } + return timing?.announce_problem ?? null; +}; + /** Red at or past the unsatisfied limit, amber within 10% of it. */ export const unsatTone = (count: number | null, limit: number | null): string | undefined => { if (count === null || limit === null) return undefined; diff --git a/tests/conftest.py b/tests/conftest.py index fee6198f4..3234940b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,8 @@ # Add the project root to Python path sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import time + import pytest @@ -39,6 +41,22 @@ def _clear_torrent_fetch_cache(): clear_torrent_fetch_cache() +@pytest.fixture(autouse=True) +def _isolate_mam_state(monkeypatch, tmp_path): + """No torrent client read or MAM download freeze carries over between tests.""" + from shelfmark.release_sources.prowlarr import mam_account, mam_freeze + + monkeypatch.setattr(mam_account, "_client_cache", None) + # The test config can name a torrent client that isn't there: read none unless a + # test supplies one (see tests/prowlarr/test_mam_unsat_timing.py). + monkeypatch.setattr( + mam_account, + "_read_client", + lambda: mam_account._ClientRead(time.time(), None, None, "No torrent client in tests"), + ) + monkeypatch.setattr(mam_freeze, "_path", lambda: tmp_path / "mam_freeze.json") + + @pytest.fixture def sample_prowlarr_result(): """Sample Prowlarr API search result.""" diff --git a/tests/download/test_wait_for_unsat_slot.py b/tests/download/test_wait_for_unsat_slot.py index 5a28f9de8..2bcc7700f 100644 --- a/tests/download/test_wait_for_unsat_slot.py +++ b/tests/download/test_wait_for_unsat_slot.py @@ -24,9 +24,15 @@ def queue(monkeypatch): @pytest.fixture def slots(monkeypatch): - """Free unsatisfied slots MAM reports; None means unknown.""" - state = {"free": 0, "enabled": True} - monkeypatch.setattr(mam_account, "unsat_free_slots", lambda **_kw: state["free"]) + """Free unsatisfied slots MAM reports (None: no limit), and why grabs must wait.""" + state = {"free": 0, "enabled": True, "hold": None} + + def allowance(**_kw): + if not state["enabled"]: + return mam_account.GrabAllowance(None) + return mam_account.GrabAllowance(state["free"], state["hold"]) + + monkeypatch.setattr(mam_account, "grab_allowance", allowance) monkeypatch.setattr(mam_account, "unsat_check_enabled", lambda: state["enabled"]) return state @@ -107,12 +113,41 @@ def test_keeps_waiting_when_mam_cannot_be_read_and_goes_when_the_check_is_off( ): monkeypatch.setattr(orchestrator, "_download_task", lambda _t, _f: None) orchestrator._process_single_download("a", _add(queue, "a")) - slots["free"] = None + slots["free"], slots["hold"] = 5, "Waiting: couldn't read the MyAnonamouse account" + assert orchestrator.release_held_downloads() == [] + assert queue.get_task("a").status_message == slots["hold"] + slots["free"], slots["hold"] = None, None # No limit reported: can't judge assert orchestrator.release_held_downloads() == [] slots["enabled"] = False assert orchestrator.release_held_downloads() == ["a"] +def test_waits_out_a_freeze_or_rejected_announces_with_free_slots(queue, slots, monkeypatch): + grabbed: list[str] = [] + monkeypatch.setattr(orchestrator, "_download_task", lambda tid, _f: grabbed.append(tid)) + slots["free"], slots["hold"] = 20, "MyAnonamouse has paused downloads on the account" + orchestrator._process_single_download("t1", _add(queue, "t1")) + + assert grabbed == [] + assert queue.get_task("t1").status_message == slots["hold"] + assert orchestrator.release_held_downloads() == [] + slots["hold"] = None # Lifted + assert orchestrator.release_held_downloads() == ["t1"] + + +def test_a_grab_refused_into_a_freeze_waits(queue, slots, monkeypatch): + def refused(_task_id, _flag): + slots["hold"] = "MyAnonamouse has paused downloads on the account" + return None + + slots["free"] = 3 + monkeypatch.setattr(orchestrator, "_download_task", refused) + orchestrator._process_single_download("t1", _add(queue, "t1")) + + assert queue.get_task_status("t1") == QueueStatus.QUEUED + assert queue.get_task("t1").status_message == slots["hold"] + + def test_a_cancelled_wait_is_dropped(queue, slots, monkeypatch): monkeypatch.setattr(orchestrator, "_download_task", lambda _t, _f: None) orchestrator._process_single_download("a", _add(queue, "a")) diff --git a/tests/prowlarr/test_mam_account.py b/tests/prowlarr/test_mam_account.py index 676cb4d49..11ce8805c 100644 --- a/tests/prowlarr/test_mam_account.py +++ b/tests/prowlarr/test_mam_account.py @@ -285,6 +285,8 @@ def test_purchase_rejects_bad_amount(self, main_module, mam): ("get", "/api/mam/account"), ("get", "/api/mam/status"), ("post", "/api/mam/upload-credit"), + ("post", "/api/mam/freeze"), + ("delete", "/api/mam/freeze"), ], ) def test_admin_only(self, main_module, mam, auth_required, method, path): @@ -293,6 +295,19 @@ def test_admin_only(self, main_module, mam, auth_required, method, path): assert resp.status_code == 403 assert mam.buys == [] + def test_freeze_entered_and_cleared_by_hand(self, main_module, mam, monkeypatch): + from shelfmark.release_sources.prowlarr import mam_freeze + + client = _client(main_module, is_admin=True) + assert client.post("/api/mam/freeze", json={"remaining": "soon"}).status_code == 400 + resp = client.post("/api/mam/freeze", json={"remaining": "1d 02:04:02"}) + assert resp.status_code == 200 + assert resp.get_json()["frozen_source"] == "manual" + assert mam_freeze.remaining_seconds() > 26 * 3600 + resp = client.delete("/api/mam/freeze") + assert resp.get_json()["frozen_seconds"] is None + assert mam_freeze.current() is None + def _mam_release(gib: float, *, freeleech: bool = False, torrent_id: int | None = 123) -> dict: return { diff --git a/tests/prowlarr/test_mam_unsat_safeguards.py b/tests/prowlarr/test_mam_unsat_safeguards.py new file mode 100644 index 000000000..6e8f3a69f --- /dev/null +++ b/tests/prowlarr/test_mam_unsat_safeguards.py @@ -0,0 +1,332 @@ +"""Safeguards against grabbing past MAM's unsatisfied limit when its count can't be +trusted: the torrent client's own count, rejected announces, and MAM's download freeze. +""" + +from __future__ import annotations + +import time +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from shelfmark.download.clients import TrackerTorrent +from shelfmark.release_sources.prowlarr import mam_account, mam_freeze +from tests.prowlarr.test_mam_account import USER, FakeMam + +HOUR = 3600 +REJECTED = "Unrecognized host/PassKey. (149.22.95.74) See https://s.mrd.ninja/upmm" + + +def _unsat_user(count: int, limit: int) -> dict: + return {**USER, "snatch_summary": {"unsat": {"count": count, "limit": limit}}} + + +def _seeding(hours: float, *, error: str | None = None) -> TrackerTorrent: + return TrackerTorrent("t", int(hours * HOUR), complete=True, tracker_error=error) + + +@pytest.fixture +def mam(monkeypatch, tmp_path): + from shelfmark.core.queue import book_queue + from shelfmark.release_sources.prowlarr import mam_points + + monkeypatch.setattr(mam_points, "_path", lambda: tmp_path / "points.json") + monkeypatch.setattr(mam_account, "_unsat_samples", []) + monkeypatch.setattr(mam_account, "unsat_reserve_slots", lambda: 5) + monkeypatch.setattr(book_queue, "handoff_times_since", lambda _since: []) + fake = FakeMam(_unsat_user(130, 150)) + monkeypatch.setattr(mam_account, "MamClient", fake) + monkeypatch.setattr(mam_account, "session_id", lambda: "session") + mam_account.invalidate_stats() + yield fake + mam_account.invalidate_stats() + + +@pytest.fixture +def client(monkeypatch): + """The torrent client's MAM torrents, as `client.torrents`.""" + state = SimpleNamespace(torrents=[]) + monkeypatch.setattr( + mam_account, + "_read_client", + lambda: mam_account._ClientRead(time.time(), "Deluge", list(state.torrents)), + ) + return state + + +class TestClientCount: + def test_the_incident_mam_shows_130_but_the_client_holds_150(self, mam, client): + """Announces refused for an hour: MAM's count stuck at 130 while 20 more + torrents sat in Deluge. Their slots are taken all the same.""" + client.torrents = [_seeding(1) for _ in range(150)] + assert mam_account.unsat_free_slots(refresh=True) == 150 - 5 - 150 + + def test_satisfied_torrents_in_the_client_dont_count(self, mam, client): + client.torrents = [_seeding(80) for _ in range(100)] + [_seeding(10)] * 10 + # The client's 10 are fewer than MAM's 130: MAM's count stands. + assert mam_account.unsat_free_slots(refresh=True) == 150 - 5 - 130 + + def test_downloading_torrents_count(self, mam, client): + client.torrents = [TrackerTorrent("t", 0, complete=False)] * 140 + assert mam_account.unsat_free_slots(refresh=True) == 5 + + def test_snatches_since_the_client_was_read_count(self, mam, client, monkeypatch): + from shelfmark.core.queue import book_queue + + client.torrents = [_seeding(1)] * 140 + task = SimpleNamespace(task_id="new", mam_torrent_id=9) + monkeypatch.setattr(book_queue, "handoff_times_since", lambda _s: [(time.time(), task)]) + assert mam_account.client_unsat_count(refresh=True) == 141 + assert mam_account.client_unsat_count(exclude={"new"}) == 140 + + def test_without_a_client_mam_count_stands(self, mam): + assert mam_account.client_unsat_count() is None + assert mam_account.unsat_free_slots(refresh=True) == 15 + + def test_the_header_sees_the_client_count_too(self, mam, client): + client.torrents = [_seeding(1)] * 150 + snapshot = mam_account.ratio_snapshot() + assert snapshot["unsat_count"] + snapshot["unsat_pending"] == 150 + + +class TestAnnounces: + def test_rejected_announces_are_reported(self, client): + client.torrents = [_seeding(1), _seeding(2, error=REJECTED)] + problem = mam_account.announce_problem() + assert problem is not None + assert "Deluge" in problem + assert "1 torrent:" in problem + assert "Unrecognized host/PassKey" in problem + + @pytest.mark.parametrize("error", ["timed out", "Torrent not registered with this tracker"]) + def test_other_tracker_errors_are_not(self, client, error): + client.torrents = [_seeding(1, error=error)] + assert mam_account.announce_problem() is None + + +class TestGrabAllowance: + def test_free_slots_when_all_is_well(self, mam, client): + client.torrents = [_seeding(1)] * 100 + assert mam_account.grab_allowance(refresh=True) == mam_account.GrabAllowance(15) + + def test_holds_while_announces_are_rejected(self, mam, client): + client.torrents = [_seeding(1, error=REJECTED)] + allowance = mam_account.grab_allowance(refresh=True) + assert allowance.free == 0 + assert allowance.hold is not None + assert "rejecting" in allowance.hold + + def test_holds_while_mam_cannot_be_read(self, mam): + mam.user = mam_account.MamError("Invalid session - ASN mismatch") + allowance = mam_account.grab_allowance(refresh=True) + assert allowance.free == 0 + assert allowance.hold is not None + assert "ASN mismatch" in allowance.hold + + def test_holds_through_a_freeze(self, mam): + mam_freeze.set_freeze(26 * HOUR, source="detected") + allowance = mam_account.grab_allowance(refresh=True) + assert allowance.free == 0 + assert allowance.hold is not None + assert "paused downloads" in allowance.hold + assert "25 h 59 m" in allowance.hold or "26 h 00 m" in allowance.hold + + def test_nothing_held_when_the_check_is_off(self, mam, monkeypatch): + mam_freeze.set_freeze(HOUR, source="manual") + monkeypatch.setattr(mam_account, "unsat_check_enabled", lambda: False) + assert mam_account.grab_allowance() == mam_account.GrabAllowance(None) + + +class TestFreeze: + MESSAGE = ( + "Attempted to Download Past Unsatisfied limit. Happened: 2026-10-05 15:53:57 " + "Time Till Download Allowed: 1d 02:04:02" + ) + + def test_parsed_from_mams_message(self): + assert mam_freeze.parse_refusal(self.MESSAGE) == 26 * HOUR + 4 * 60 + 2 + + def test_parsed_from_html_and_escaped_html(self): + html = "Time Till Download Allowed:23:00:00" + assert mam_freeze.parse_refusal(html) == 23 * HOUR + escaped = r'{"error":"Time Till Download Allowed: 1d 00:00:05"}' + assert mam_freeze.parse_refusal(escaped) == 24 * HOUR + 5 + + def test_a_day_when_no_time_is_given(self): + text = "Attempted to Download Past Unsatisfied limit" + assert mam_freeze.parse_refusal(text) == mam_freeze.DEFAULT_FREEZE_SECONDS + + def test_other_replies_are_not_a_freeze(self): + assert mam_freeze.parse_refusal("Not found") is None + assert mam_freeze.note_refusal("Not found") is False + assert mam_freeze.current() is None + + def test_recorded_and_lifted(self, monkeypatch): + assert mam_freeze.note_refusal(self.MESSAGE) is True + freeze = mam_freeze.current() + assert freeze is not None + assert freeze["source"] == "detected" + remaining = mam_freeze.remaining_seconds() + assert remaining is not None + assert 26 * HOUR <= remaining <= 26 * HOUR + 4 * 60 + 2 + + later = time.time() + 27 * HOUR + monkeypatch.setattr(mam_freeze.time, "time", lambda: later) + assert mam_freeze.current() is None + + def test_cleared_by_hand(self): + mam_freeze.set_freeze(HOUR, source="manual") + mam_freeze.clear() + assert mam_freeze.remaining_seconds() is None + + def test_duration_as_entered(self): + assert mam_freeze.parse_duration("1d 02:04:02") == 26 * HOUR + 242 + assert mam_freeze.parse_duration(" 5:00:00 ") == 5 * HOUR + assert mam_freeze.parse_duration("tomorrow") is None + + def test_timing_reports_the_freeze(self, monkeypatch, client): + monkeypatch.setattr(mam_account, "is_configured", lambda: True) + mam_freeze.set_freeze(2 * HOUR, source="manual") + timing = mam_account.unsat_timing() + assert timing["frozen_source"] == "manual" + assert 2 * HOUR - 5 <= timing["frozen_seconds"] <= 2 * HOUR + + +class TestRefusedDownload: + def _fetch(self, monkeypatch, status: int, body: bytes): + from shelfmark.download.clients import torrent_utils + + response = MagicMock(status_code=status, content=body, headers={}) + response.raise_for_status.side_effect = ( + None if status < 400 else torrent_utils.requests.exceptions.HTTPError("refused") + ) + monkeypatch.setattr(torrent_utils.requests, "get", lambda *_a, **_kw: response) + return torrent_utils._fetch_torrent_info("https://prowlarr/download?id=1") + + def test_a_refusal_page_records_the_freeze(self, monkeypatch): + info = self._fetch(monkeypatch, 200, TestFreeze.MESSAGE.encode()) + assert info.info_hash is None + assert mam_freeze.current() is not None + + def test_an_error_reply_records_the_freeze(self, monkeypatch): + info = self._fetch(monkeypatch, 500, TestFreeze.MESSAGE.encode()) + assert info.fetch_error + assert mam_freeze.current() is not None + + def test_other_bad_replies_dont(self, monkeypatch): + self._fetch(monkeypatch, 404, b"Not found") + assert mam_freeze.current() is None + + +class TestClientsReportRejectedAnnounces: + def test_deluge(self, monkeypatch): + from shelfmark.download.clients.deluge import DelugeClient + + deluge = DelugeClient.__new__(DelugeClient) + monkeypatch.setattr(deluge, "_ensure_connected", lambda: None) + statuses = { + "a": { + "name": "A", + "tracker_host": "myanonamouse.net", + "seeding_time": 60, + "is_finished": True, + "tracker_status": f"Error: {REJECTED}", + }, + "b": { + "name": "B", + "tracker_host": "myanonamouse.net", + "seeding_time": 60, + "is_finished": True, + "tracker_status": "Announce OK", + }, + } + monkeypatch.setattr(deluge, "_rpc_call", MagicMock(return_value=statuses)) + assert [t.tracker_error for t in deluge.list_tracker_torrents("myanonamouse")] == [ + REJECTED, + None, + ] + + def test_qbittorrent_finds_mam_torrents_with_no_working_tracker(self, monkeypatch): + from shelfmark.download.clients import qbittorrent + from shelfmark.download.clients.qbittorrent import QBittorrentClient + + monkeypatch.setattr(qbittorrent, "_on_tracker", {}) + + qbit = QBittorrentClient.__new__(QBittorrentClient) + qbit._base_url = "http://qbit" + qbit._api_key = "key" + records = [ + SimpleNamespace(name="A", hash="a", tracker="", seeding_time=60, progress=1.0), + SimpleNamespace(name="B", hash="b", tracker="", seeding_time=60, progress=1.0), + ] + rows = { + "a": [ + {"url": "** [DHT] **", "status": 0, "msg": ""}, + {"url": "https://t.myanonamouse.net/announce", "status": 4, "msg": REJECTED}, + ], + "b": [{"url": "https://other.org/announce", "status": 4, "msg": "down"}], + } + + def get(_url, params, timeout): + return SimpleNamespace(raise_for_status=lambda: None, json=lambda: rows[params["hash"]]) + + monkeypatch.setattr(qbit, "_request_torrent_info_records", lambda _p: (records, None)) + qbit._client = SimpleNamespace(_session=SimpleNamespace(get=get)) + assert qbit.list_tracker_torrents("myanonamouse") == [ + TrackerTorrent("A", 60, True, REJECTED) + ] + + def test_transmission(self): + from shelfmark.download.clients.transmission import TransmissionClient + + transmission = TransmissionClient.__new__(TransmissionClient) + url = "https://t.myanonamouse.net/a" + stats = SimpleNamespace( + announce=url, + has_announced=True, + last_announce_succeeded=False, + last_announce_result=REJECTED, + ) + transmission._client = SimpleNamespace( + get_torrents=lambda arguments: [ + SimpleNamespace( + name="A", + seconds_seeding=60, + percent_done=1.0, + trackers=[SimpleNamespace(announce=url)], + tracker_stats=[stats], + ) + ] + ) + assert transmission.list_tracker_torrents("myanonamouse") == [ + TrackerTorrent("A", 60, True, REJECTED) + ] + + def test_qbittorrent_asks_once_about_other_and_idle_torrents(self, monkeypatch): + from shelfmark.download.clients import qbittorrent + from shelfmark.download.clients.qbittorrent import QBittorrentClient + + monkeypatch.setattr(qbittorrent, "_on_tracker", {}) + qbit = QBittorrentClient.__new__(QBittorrentClient) + qbit._base_url = "http://qbit" + qbit._api_key = "key" + records = [ + SimpleNamespace(name="O", hash="o", tracker="", state="stalledUP", progress=1.0), + SimpleNamespace(name="P", hash="p", tracker="", state="stoppedUP", progress=1.0), + ] + rows = { + "o": [{"url": "https://other.org/announce", "status": 4, "msg": "down"}], + "p": [{"url": "https://t.myanonamouse.net/announce", "status": 0, "msg": ""}], + } + asked: list[str] = [] + + def get(_url, params, timeout): + asked.append(params["hash"]) + return SimpleNamespace(raise_for_status=lambda: None, json=lambda: rows[params["hash"]]) + + monkeypatch.setattr(qbit, "_request_torrent_info_records", lambda _p: (records, None)) + qbit._client = SimpleNamespace(_session=SimpleNamespace(get=get)) + for _ in range(3): + assert [t.name for t in qbit.list_tracker_torrents("myanonamouse")] == ["P"] + assert sorted(asked) == ["o", "p"] diff --git a/tests/prowlarr/test_mam_unsat_timing.py b/tests/prowlarr/test_mam_unsat_timing.py index b9ab30f5a..b59c2295a 100644 --- a/tests/prowlarr/test_mam_unsat_timing.py +++ b/tests/prowlarr/test_mam_unsat_timing.py @@ -41,9 +41,12 @@ def test_nothing_waiting(): assert timing["within_window"] == 0 +_READ_CLIENT = mam_account._read_client # The real one; tests read no client by default + + def _use_client(monkeypatch, client): monkeypatch.setattr(mam_account, "is_configured", lambda: True) - monkeypatch.setattr(mam_account, "_timing_cache", None) + monkeypatch.setattr(mam_account, "_read_client", _READ_CLIENT) monkeypatch.setattr("shelfmark.download.clients.get_client", lambda protocol: client)