feat: matilde_fetch_fulltext — legal open-access full-text locator

README.md

@@ -55,10 +55,12 @@ score and an evidence trail, never a false certainty.
 | `matilde_openneuro_dataset_info` | Metadata for an OpenNeuro dataset (title, authors, modalities, subjects, tasks, size) |
 | `matilde_openneuro_search` | List OpenNeuro dataset IDs to discover brain-imaging datasets |
 | `matilde_openneuro_list_files` | List a dataset's files with sizes and direct download URLs |
+| `matilde_fetch_fulltext` | Resolve the best **legal open-access** full text for a DOI (PDF or OA landing) via OpenAlex/Unpaywall/arXiv — for grounding a claim against the source, not just its metadata. Never routes around a paywall |
 
-No API keys required — Crossref, OpenAlex, and DataCite are free and
-unauthenticated. Optionally set `MATILDE_CONTACT_EMAIL` to join the providers'
-polite pools.
+No API keys required — Crossref, OpenAlex, DataCite, and the open-access
+providers (OpenAlex, Unpaywall, arXiv) are free. Optionally set
+`MATILDE_CONTACT_EMAIL` to join the providers' polite pools and enable the
+Unpaywall lookup (which widens open-access coverage).
 
 ## Try it
 

hermes-skill/SKILL.md

@@ -82,11 +82,26 @@ For a fast retraction-only check by DOI:
 matilde_check_retraction(doi="10.1016/...")
 ```
 
+To read the actual paper — to ground a claim against what the source really
+says — resolve its **legal open-access** full text by DOI:
+
+```
+matilde_fetch_fulltext(doi="10.7554/eLife.00013")
+```
+
+It returns the best open-access location (a direct PDF where one exists, else an
+OA landing page) from OpenAlex / Unpaywall / arXiv, with `is_oa`, `oa_status`,
+and license. If no open-access copy exists, `is_oa` is false and no URL is
+returned — it surfaces only legal OA sources and will not route around a
+paywall. Set `MATILDE_CONTACT_EMAIL` to widen coverage via Unpaywall. When the
+result gives a URL, fetch and read it with your general research/file tools.
+
 **Interpreting axes honestly.** A `verified` verdict means *checked against
 authoritative metadata* — existence, identity, retraction status. It does **not**
 mean the cited passage actually supports the claim it is attached to. That deeper
-check (claim-support grounding) is not yet automated; when it matters, read the
-source and confirm the passage yourself, and say that you did.
+check (claim-support grounding) is not yet automated; when it matters, pull the
+source with `matilde_fetch_fulltext`, read the passage, confirm it yourself, and
+say that you did.
 
 ### Phase 4: Synthesize
 

matilde_plugin/__init__.py

@@ -54,13 +54,15 @@
 OPENNEURO_INFO_SCHEMA = _tools_mod.OPENNEURO_INFO_SCHEMA
 OPENNEURO_SEARCH_SCHEMA = _tools_mod.OPENNEURO_SEARCH_SCHEMA
 OPENNEURO_FILES_SCHEMA = _tools_mod.OPENNEURO_FILES_SCHEMA
+FETCH_FULLTEXT_SCHEMA = _tools_mod.FETCH_FULLTEXT_SCHEMA
 _check_available = _tools_mod._check_available
 _handle_verify_citation = _tools_mod._handle_verify_citation
 _handle_verify_bibliography = _tools_mod._handle_verify_bibliography
 _handle_check_retraction = _tools_mod._handle_check_retraction
 _handle_openneuro_dataset_info = _tools_mod._handle_openneuro_dataset_info
 _handle_openneuro_search = _tools_mod._handle_openneuro_search
 _handle_openneuro_list_files = _tools_mod._handle_openneuro_list_files
+_handle_fetch_fulltext = _tools_mod._handle_fetch_fulltext
 
 # ---------------------------------------------------------------------------
 # Tool registry — (tool_name, schema, handler, emoji)
@@ -72,6 +74,7 @@
     ("matilde_openneuro_dataset_info", OPENNEURO_INFO_SCHEMA, _handle_openneuro_dataset_info, "🧠"),
     ("matilde_openneuro_search", OPENNEURO_SEARCH_SCHEMA, _handle_openneuro_search, "🔎"),
     ("matilde_openneuro_list_files", OPENNEURO_FILES_SCHEMA, _handle_openneuro_list_files, "🗂"),
+    ("matilde_fetch_fulltext", FETCH_FULLTEXT_SCHEMA, _handle_fetch_fulltext, "📄"),
 )
 
 

matilde_plugin/engine/fulltext.py

@@ -0,0 +1,164 @@
+"""Open-access full-text locator.
+
+Given a DOI, resolve the best *legal* open-access location for the work — a
+direct PDF where one exists, otherwise an OA landing page. Sources, in priority
+order, are all free and unauthenticated (or email-gated, never key-gated):
+
+  1. **OpenAlex** — ``open_access`` + ``best_oa_location`` (no email needed).
+  2. **Unpaywall** — queried only when a contact email is supplied (its API
+     requires ``?email=``). Often finds a publisher/repository OA copy OpenAlex
+     hasn't indexed yet.
+  3. **arXiv synthesis** — an arXiv-registered DOI (``10.48550/arXiv.<id>``)
+     always has a legal PDF at ``arxiv.org/pdf/<id>``, even if the providers miss.
+
+This locator only ever returns open-access sources. A paywalled work resolves to
+``is_oa=False`` with no URL — it deliberately does **not** route around a
+paywall. The point is to give a citation-verifier the *content* it can legally
+read to ground a claim, not to bypass access controls.
+
+Design mirrors ``citations``: all network I/O is injected via a ``fetch`` callable
+``(url) -> parsed JSON dict`` (raising ``LookupError`` on 404), so every path is
+unit-testable without a network. ``default_fetch`` is the stdlib production default.
+"""
+from __future__ import annotations
+
+import dataclasses
+import re
+import urllib.parse
+from dataclasses import dataclass, field
+from typing import Callable, Optional
+
+from .citations import OPENALEX_BASE, _normalize_doi, default_fetch
+
+FetchFn = Callable[..., dict]
+
+UNPAYWALL_BASE = "https://api.unpaywall.org/v2"
+# arXiv-registered DOIs look like 10.48550/arXiv.1706.03762 (case-insensitive).
+_ARXIV_DOI = re.compile(r"^10\.48550/arxiv\.(.+)$", re.I)
+
+
+# ---------------------------------------------------------------------------
+# Data model
+# ---------------------------------------------------------------------------
+
+@dataclass
+class FullTextResult:
+    """The resolved best open-access location for a DOI (or a closed verdict)."""
+
+    doi: str
+    is_oa: bool = False
+    oa_status: str = "unknown"   # gold | green | hybrid | bronze | closed | unknown
+    best_url: str = ""           # best PDF if available, else best landing page
+    pdf_url: str = ""
+    landing_url: str = ""
+    source: str = ""             # which provider produced the chosen location
+    license: str = ""
+    candidates: list = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return dataclasses.asdict(self)
+
+
+# ---------------------------------------------------------------------------
+# Provider parsers (each returns a normalized loc dict, or None on miss)
+# ---------------------------------------------------------------------------
+
+def _try_openalex(doi: str, fetch: FetchFn) -> Optional[dict]:
+    try:
+        work = fetch(f"{OPENALEX_BASE}/works/doi:{doi}")
+    except Exception:
+        return None
+    oa = work.get("open_access") or {}
+    best = work.get("best_oa_location") or {}
+    # Only ``pdf_url`` is a guaranteed direct PDF. ``oa_url`` is the "best OA URL"
+    # but is often a landing page — treat it as a landing fallback, never a PDF.
+    return {
+        "is_oa": bool(oa.get("is_oa")),
+        "oa_status": oa.get("oa_status") or "",
+        "pdf_url": best.get("pdf_url") or "",
+        "landing_url": best.get("landing_page_url") or oa.get("oa_url") or "",
+        "license": best.get("license") or "",
+        "version": best.get("version") or "",
+        "host_type": (best.get("source") or {}).get("type") or "",
+    }
+
+
+def _try_unpaywall(doi: str, fetch: FetchFn, email: str) -> Optional[dict]:
+    email_q = urllib.parse.quote(email)
+    try:
+        data = fetch(f"{UNPAYWALL_BASE}/{doi}?email={email_q}")
+    except Exception:
+        return None
+    best = data.get("best_oa_location") or {}
+    return {
+        "is_oa": bool(data.get("is_oa")),
+        "oa_status": data.get("oa_status") or "",
+        "pdf_url": best.get("url_for_pdf") or "",
+        "landing_url": best.get("url") or "",
+        "license": best.get("license") or "",
+        "version": best.get("version") or "",
+        "host_type": best.get("host_type") or "",
+    }
+
+
+def _arxiv_pdf(doi: str) -> str:
+    m = _ARXIV_DOI.match(doi)
+    return f"https://arxiv.org/pdf/{m.group(1)}" if m else ""
+
+
+# ---------------------------------------------------------------------------
+# Assembly
+# ---------------------------------------------------------------------------
+
+def _from_loc(doi: str, loc: dict, source: str) -> FullTextResult:
+    pdf = loc.get("pdf_url") or ""
+    landing = loc.get("landing_url") or ""
+    best = pdf or landing
+    candidate = {
+        "url": best, "pdf_url": pdf, "landing_url": landing,
+        "license": loc.get("license") or "", "version": loc.get("version") or "",
+        "host_type": loc.get("host_type") or "", "source": source,
+    }
+    return FullTextResult(
+        doi=doi, is_oa=True, oa_status=loc.get("oa_status") or "unknown",
+        best_url=best, pdf_url=pdf, landing_url=landing, source=source,
+        license=loc.get("license") or "", candidates=[candidate],
+    )
+
+
+def find_open_access(doi: str, fetch: Optional[FetchFn] = None, *,
+                     email: Optional[str] = None) -> FullTextResult:
+    """Resolve the best legal open-access location for *doi*.
+
+    Returns a :class:`FullTextResult`. ``is_oa=False`` means no open-access copy
+    was found — the work is paywalled and this locator will not route around it.
+    Pass *email* to enable the Unpaywall lookup (its API requires a contact
+    address); without it, only OpenAlex + arXiv are consulted.
+    """
+    fetch = fetch or default_fetch
+    doi = _normalize_doi(doi)
+    result = FullTextResult(doi=doi)
+    if not doi:
+        return result
+
+    # 1. OpenAlex (primary, no email needed)
+    oa = _try_openalex(doi, fetch)
+    if oa and oa["is_oa"] and (oa["pdf_url"] or oa["landing_url"]):
+        return _from_loc(doi, oa, "openalex")
+    if oa is not None:
+        result.is_oa = oa["is_oa"]
+        result.oa_status = oa["oa_status"] or "closed"
+
+    # 2. Unpaywall (only with a contact email)
+    if email:
+        up = _try_unpaywall(doi, fetch, email)
+        if up and up["is_oa"] and (up["pdf_url"] or up["landing_url"]):
+            return _from_loc(doi, up, "unpaywall")
+
+    # 3. arXiv synthesis — a registered arXiv DOI always has a legal PDF
+    arx = _arxiv_pdf(doi)
+    if arx:
+        return _from_loc(doi, {"pdf_url": arx, "oa_status": "green",
+                               "host_type": "repository"}, "arxiv")
+
+    return result

matilde_plugin/plugin.yaml

@@ -11,7 +11,7 @@ name: "matilde"
 
 version: "0.1.0"
 
-description: "Matilde — academic research assistant: verifiable citations (Crossref/OpenAlex/DataCite + retraction) and OpenNeuro/BIDS dataset discovery."
+description: "Matilde — academic research assistant: verifiable citations (Crossref/OpenAlex/DataCite + retraction), legal open-access full-text retrieval (OpenAlex/Unpaywall/arXiv), and OpenNeuro/BIDS dataset discovery."
 
 author: "NimbleCoAI"
 
@@ -28,3 +28,4 @@ provides_tools:
   - "matilde_openneuro_dataset_info"
   - "matilde_openneuro_search"
   - "matilde_openneuro_list_files"
+  - "matilde_fetch_fulltext"

matilde_plugin/tools.py

@@ -370,3 +370,55 @@ def _handle_openneuro_list_files(args: dict, **kwargs: Any) -> str:
         )
     except Exception as exc:
         return _tool_error(f"openneuro_list_files failed: {type(exc).__name__}: {exc}")
+
+
+# ---------------------------------------------------------------------------
+# Tool: open-access full-text locator
+# ---------------------------------------------------------------------------
+
+FETCH_FULLTEXT_SCHEMA = {
+    "name": "matilde_fetch_fulltext",
+    "description": (
+        "Find the best *legal open-access* full-text location for a paper by DOI "
+        "— a direct PDF where one exists, else an OA landing page — using OpenAlex, "
+        "Unpaywall, and arXiv. Use this to retrieve the actual paper content so a "
+        "claim can be grounded against the source, not just its metadata. Returns "
+        "is_oa, oa_status (gold/green/hybrid/bronze/closed), pdf_url, landing_url, "
+        "license, and the provider. If no open-access copy exists, is_oa is false "
+        "and no URL is returned — this tool only surfaces legal OA sources and will "
+        "not route around a paywall. Set MATILDE_CONTACT_EMAIL (or pass 'email') to "
+        "enable the Unpaywall lookup, which widens coverage."
+    ),
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "doi": {"type": "string", "description": "DOI of the work (bare, or as a doi.org URL)."},
+            "email": {"type": "string", "description": "Contact email to enable the Unpaywall lookup (optional; falls back to MATILDE_CONTACT_EMAIL)."},
+        },
+        "required": ["doi"],
+    },
+}
+
+
+def _handle_fetch_fulltext(args: dict, **kwargs: Any) -> str:
+    doi = str(args.get("doi", "")).strip()
+    if not doi:
+        return _tool_error("'doi' is required (bare or doi.org URL).")
+    try:
+        import os
+        from .engine.fulltext import find_open_access
+        email = (str(args.get("email", "")).strip()
+                 or os.environ.get("MATILDE_CONTACT_EMAIL", "").strip()
+                 or None)
+        res = find_open_access(doi, email=email)
+        payload = res.to_dict()
+        if res.is_oa:
+            payload["message"] = (f"Open access ({res.oa_status}) via {res.source}: "
+                                  f"{res.best_url}")
+        else:
+            payload["message"] = (f"No open-access copy found for {res.doi} "
+                                  f"(status: {res.oa_status}). Matilde returns only "
+                                  f"legal OA sources.")
+        return _tool_result(payload)
+    except Exception as exc:
+        return _tool_error(f"fetch_fulltext failed: {type(exc).__name__}: {exc}")