feat(fulltext): provider-neutral external resolver hook (opt-in, last resort)

README.md

@@ -62,6 +62,23 @@ providers (OpenAlex, Unpaywall, arXiv) are free. Optionally set
 `MATILDE_CONTACT_EMAIL` to join the providers' polite pools and enable the
 Unpaywall lookup (which widens open-access coverage).
 
+### Optional: an external full-text resolver
+
+`matilde_fetch_fulltext` has a provider-neutral extension point. If you set
+`MATILDE_FULLTEXT_RESOLVER_URL`, then **only when every open-access lookup has
+missed**, Matilde will ask that endpoint for a full-text URL:
+
+```
+GET {MATILDE_FULLTEXT_RESOLVER_URL}/resolve?doi=<doi>  ->  {"pdf_url": "..."}
+```
+
+This package ships no such service and names no provider — you supply one out of
+band (an institutional/library proxy, a licensed full-text API, a self-hosted
+service). A result obtained this way is reported with `is_oa: false` and
+`source: "external-resolver"`: it is full text, **not** open access, and you are
+responsible for having the right to access it. Leave the variable unset and the
+behaviour never engages.
+
 ## Try it
 
 ```python

matilde_plugin/engine/fulltext.py

@@ -106,6 +106,25 @@ def _arxiv_pdf(doi: str) -> str:
     return f"https://arxiv.org/pdf/{m.group(1)}" if m else ""
 
 
+def _try_external_resolver(doi: str, fetch: FetchFn, resolver_url: str) -> str:
+    """Ask a configured external full-text resolver for a PDF URL.
+
+    This is a provider-neutral extension point: ``resolver_url`` points at a
+    self-hosted or third-party service implementing the contract
+    ``GET {resolver_url}/resolve?doi={doi} -> {"pdf_url": "...", ...}``. This
+    package ships no such service and names no provider — the operator supplies
+    one out of band (an institutional proxy, a paid API, anything). Consulted
+    only after every legal open-access lookup has missed.
+    """
+    base = resolver_url.rstrip("/")
+    doi_q = urllib.parse.quote(doi, safe="")
+    try:
+        data = fetch(f"{base}/resolve?doi={doi_q}")
+    except Exception:
+        return ""
+    return (data or {}).get("pdf_url") or ""
+
+
 # ---------------------------------------------------------------------------
 # Assembly
 # ---------------------------------------------------------------------------
@@ -127,13 +146,20 @@ def _from_loc(doi: str, loc: dict, source: str) -> FullTextResult:
 
 
 def find_open_access(doi: str, fetch: Optional[FetchFn] = None, *,
-                     email: Optional[str] = None) -> FullTextResult:
-    """Resolve the best legal open-access location for *doi*.
-
-    Returns a :class:`FullTextResult`. ``is_oa=False`` means no open-access copy
-    was found — the work is paywalled and this locator will not route around it.
-    Pass *email* to enable the Unpaywall lookup (its API requires a contact
-    address); without it, only OpenAlex + arXiv are consulted.
+                     email: Optional[str] = None,
+                     resolver_url: Optional[str] = None) -> FullTextResult:
+    """Resolve the best full-text location for *doi*, preferring legal open access.
+
+    Returns a :class:`FullTextResult`. Legal open-access sources are tried first
+    (OpenAlex, then Unpaywall if *email* is given, then an arXiv-DOI synthesis).
+    ``is_oa=False`` means no open-access copy was found.
+
+    If — and only if — every OA lookup misses and *resolver_url* is configured,
+    a provider-neutral external resolver is consulted as a last resort. A hit
+    there populates the full-text URL but keeps ``is_oa=False`` and
+    ``source="external-resolver"``: it is full text, not open access, and the
+    result says so. A legal OA copy always wins, so a configured resolver is
+    never consulted when an open-access copy exists.
     """
     fetch = fetch or default_fetch
     doi = _normalize_doi(doi)
@@ -161,4 +187,15 @@ def find_open_access(doi: str, fetch: Optional[FetchFn] = None, *,
         return _from_loc(doi, {"pdf_url": arx, "oa_status": "green",
                                "host_type": "repository"}, "arxiv")
 
+    # 4. External resolver (opt-in, last resort) — full text, NOT open access.
+    if resolver_url:
+        pdf = _try_external_resolver(doi, fetch, resolver_url)
+        if pdf:
+            result.pdf_url = pdf
+            result.best_url = pdf
+            result.source = "external-resolver"
+            result.candidates = [{"url": pdf, "pdf_url": pdf, "landing_url": "",
+                                  "license": "", "version": "", "host_type": "",
+                                  "source": "external-resolver"}]
+
     return result

matilde_plugin/tools.py

@@ -384,16 +384,20 @@ def _handle_openneuro_list_files(args: dict, **kwargs: Any) -> str:
         "Unpaywall, and arXiv. Use this to retrieve the actual paper content so a "
         "claim can be grounded against the source, not just its metadata. Returns "
         "is_oa, oa_status (gold/green/hybrid/bronze/closed), pdf_url, landing_url, "
-        "license, and the provider. If no open-access copy exists, is_oa is false "
-        "and no URL is returned — this tool only surfaces legal OA sources and will "
-        "not route around a paywall. Set MATILDE_CONTACT_EMAIL (or pass 'email') to "
-        "enable the Unpaywall lookup, which widens coverage."
+        "license, and the provider. If no open-access copy exists, is_oa is false. "
+        "Set MATILDE_CONTACT_EMAIL (or pass 'email') to enable the Unpaywall lookup, "
+        "which widens coverage. If an external full-text resolver is configured "
+        "(MATILDE_FULLTEXT_RESOLVER_URL or 'resolver_url'), it is consulted only as "
+        "a last resort after every open-access lookup misses; such a result is "
+        "flagged is_oa=false with source 'external-resolver' — it is full text, not "
+        "open access, so confirm you have the right to access it."
     ),
     "parameters": {
         "type": "object",
         "properties": {
             "doi": {"type": "string", "description": "DOI of the work (bare, or as a doi.org URL)."},
             "email": {"type": "string", "description": "Contact email to enable the Unpaywall lookup (optional; falls back to MATILDE_CONTACT_EMAIL)."},
+            "resolver_url": {"type": "string", "description": "Base URL of an external full-text resolver to use as a last resort (optional; falls back to MATILDE_FULLTEXT_RESOLVER_URL). Off unless configured."},
         },
         "required": ["doi"],
     },
@@ -410,15 +414,22 @@ def _handle_fetch_fulltext(args: dict, **kwargs: Any) -> str:
         email = (str(args.get("email", "")).strip()
                  or os.environ.get("MATILDE_CONTACT_EMAIL", "").strip()
                  or None)
-        res = find_open_access(doi, email=email)
+        resolver_url = (str(args.get("resolver_url", "")).strip()
+                        or os.environ.get("MATILDE_FULLTEXT_RESOLVER_URL", "").strip()
+                        or None)
+        res = find_open_access(doi, email=email, resolver_url=resolver_url)
         payload = res.to_dict()
         if res.is_oa:
             payload["message"] = (f"Open access ({res.oa_status}) via {res.source}: "
                                   f"{res.best_url}")
+        elif res.best_url:
+            # Full text via a configured external resolver — NOT open access.
+            payload["message"] = (f"Full text via {res.source} (NOT open access — "
+                                  f"verify you have the right to access it): "
+                                  f"{res.best_url}")
         else:
             payload["message"] = (f"No open-access copy found for {res.doi} "
-                                  f"(status: {res.oa_status}). Matilde returns only "
-                                  f"legal OA sources.")
+                                  f"(status: {res.oa_status}).")
         return _tool_result(payload)
     except Exception as exc:
         return _tool_error(f"fetch_fulltext failed: {type(exc).__name__}: {exc}")

tests/test_fulltext.py

@@ -168,6 +168,63 @@ def test_arxiv_doi_synthesizes_pdf_url_when_providers_miss():
     assert res.source == "arxiv"
 
 
+# ---------------------------------------------------------------------------
+# External resolver hook (provider-neutral; opt-in via resolver_url)
+# ---------------------------------------------------------------------------
+
+# A configured external resolver answers with a full-text URL for a paywalled
+# work. Its OA status is unknown/none — the result must NOT claim open access.
+EXTERNAL_HIT = {"pdf_url": "https://resolver.example.net/files/closed-paper.pdf"}
+
+
+def test_external_resolver_used_when_oa_misses():
+    fetch = make_fetch({
+        "api.openalex.org": OPENALEX_CLOSED,
+        "resolver.example.net": EXTERNAL_HIT,
+    })
+    res = find_open_access("10.1234/closed-paper", fetch=fetch,
+                           resolver_url="https://resolver.example.net")
+    assert res.best_url == "https://resolver.example.net/files/closed-paper.pdf"
+    assert res.pdf_url == "https://resolver.example.net/files/closed-paper.pdf"
+    assert res.source == "external-resolver"
+    # Honesty: an external resolver is NOT open access — don't mislabel it.
+    assert res.is_oa is False
+
+
+def test_external_resolver_skipped_when_unset():
+    fetch = make_fetch({
+        "api.openalex.org": OPENALEX_CLOSED,
+        "resolver.example.net": EXTERNAL_HIT,
+    })
+    res = find_open_access("10.1234/closed-paper", fetch=fetch, resolver_url=None)
+    assert res.best_url == ""
+    assert not any("resolver.example.net" in u for u in fetch.calls)
+
+
+def test_external_resolver_not_consulted_when_legal_oa_exists():
+    # The legal OA copy wins; the external resolver (maybe Sci-Hub) is never hit.
+    fetch = make_fetch({
+        "api.openalex.org": OPENALEX_OA,
+        "resolver.example.net": EXTERNAL_HIT,
+    })
+    res = find_open_access("10.1234/oa-paper", fetch=fetch,
+                           resolver_url="https://resolver.example.net")
+    assert res.source == "openalex"
+    assert res.is_oa is True
+    assert not any("resolver.example.net" in u for u in fetch.calls)
+
+
+def test_external_resolver_miss_falls_through_to_closed():
+    fetch = make_fetch({
+        "api.openalex.org": OPENALEX_CLOSED,
+        "resolver.example.net": {"pdf_url": None},
+    })
+    res = find_open_access("10.1234/closed-paper", fetch=fetch,
+                           resolver_url="https://resolver.example.net")
+    assert res.best_url == ""
+    assert res.is_oa is False
+
+
 # ---------------------------------------------------------------------------
 # DOI normalization
 # ---------------------------------------------------------------------------

tests/test_fulltext_tool.py

@@ -56,3 +56,32 @@ def boom(*a, **k):
     out = json.loads(_handle_fetch_fulltext({"doi": "10.1234/x"}))
     assert out["success"] is False
     assert "fetch_fulltext failed" in out["error"]
+
+
+def test_handler_forwards_resolver_url_from_env(monkeypatch):
+    captured = {}
+
+    def capture(doi, **kwargs):
+        captured.update(kwargs)
+        return ft.FullTextResult(doi=doi)
+
+    monkeypatch.setattr(ft, "find_open_access", capture)
+    monkeypatch.setenv("MATILDE_FULLTEXT_RESOLVER_URL", "https://resolver.example.net")
+
+    _handle_fetch_fulltext({"doi": "10.1234/x"})
+    assert captured.get("resolver_url") == "https://resolver.example.net"
+
+
+def test_handler_message_flags_external_as_not_open_access(monkeypatch):
+    fake = ft.FullTextResult(
+        doi="10.1234/x", is_oa=False, oa_status="closed",
+        best_url="https://resolver/x.pdf", pdf_url="https://resolver/x.pdf",
+        source="external-resolver",
+    )
+    monkeypatch.setattr(ft, "find_open_access", lambda *a, **k: fake)
+
+    out = json.loads(_handle_fetch_fulltext({"doi": "10.1234/x"}))
+    assert out["success"] is True
+    assert out["pdf_url"] == "https://resolver/x.pdf"
+    assert out["is_oa"] is False
+    assert "not open access" in out["message"].lower()