Bibliography ingestion (BibTeX/DOI) + verify CLI

README.md

@@ -70,6 +70,31 @@ result = verify_reference(ref)
 print(result.verdict, result.score)   # -> e.g. "verified" 1.0
 ```
 
+### Command line — verify a whole bibliography
+
+```bash
+python3 -m engine.cli refs.bib                  # verify a BibTeX file
+python3 -m engine.cli dois.txt                  # or a list of DOIs (one per line)
+python3 -m engine.cli --doi 10.1038/171737a0    # or a single DOI
+python3 -m engine.cli refs.bib --json           # machine-readable
+```
+
+Output for a mixed bibliography:
+
+```
+Verified 3 reference(s):
+  [OK ] verified      0.9  Molecular structure of nucleic acids
+  [RET] retracted     0.1  Ileal-lymphoid-nodular hyperplasia
+  [XX ] not_found     0.1  A Totally Fabricated Paper
+
+not_found=1  retracted=1  verified=1
+```
+
+Exit code is non-zero if any reference is `not_found` or `retracted` — so you can
+drop it into CI or a pre-commit hook on a manuscript's `.bib`.
+
+### Tests
+
 ```bash
 python3 -m pytest tests/ --ignore=tests/test_citations_integration.py   # offline unit suite
 MATILDE_LIVE=1 python3 -m pytest tests/test_citations_integration.py    # live API checks

engine/cli.py

@@ -0,0 +1,117 @@
+"""Matilde command-line interface — verify a bibliography from the terminal.
+
+Usage::
+
+    python3 -m engine.cli refs.bib                 # verify a BibTeX file
+    python3 -m engine.cli dois.txt                 # verify a list of DOIs
+    python3 -m engine.cli --doi 10.1038/171737a0   # verify one DOI
+    python3 -m engine.cli refs.bib --json          # machine-readable output
+    python3 -m engine.cli refs.bib --email you@uni.edu   # polite-pool contact
+
+Exit codes: 0 = all references verified / only warnings; 1 = at least one
+``not_found`` or ``retracted`` reference (useful as a pre-commit / CI gate on a
+manuscript's .bib); 2 = usage error.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from typing import Callable, Optional
+
+from engine.citations import Reference, verify_reference
+from engine.parsing import parse_bibtex, parse_dois
+
+_VERDICT_MARK = {
+    "verified": "OK ",
+    "warnings": "!  ",
+    "unverifiable": "?  ",
+    "not_found": "XX ",
+    "retracted": "RET",
+}
+
+
+def load_references(text: str, hint: str = "") -> list:
+    """Load references from *text*, choosing BibTeX vs DOI-list by content/hint."""
+    if "@" in text and "=" in text or hint.endswith(".bib"):
+        refs = parse_bibtex(text)
+        if refs:
+            return refs
+    return parse_dois(text)
+
+
+def _format_text(results: list) -> str:
+    lines, summary = [], {}
+    flagged = []
+    for i, r in enumerate(results):
+        summary[r.verdict] = summary.get(r.verdict, 0) + 1
+        label = r.reference.title or r.reference.doi or r.reference.raw[:50] or "(no id)"
+        mark = _VERDICT_MARK.get(r.verdict, "   ")
+        lines.append(f"  [{mark}] {r.verdict:<12} {r.score:>4}  {label}")
+        if r.verdict in ("not_found", "retracted"):
+            flagged.append((i, r.verdict, label))
+    header = f"Verified {len(results)} reference(s):"
+    tally = "  ".join(f"{k}={v}" for k, v in sorted(summary.items()))
+    out = [header, *lines, "", tally]
+    if flagged:
+        out.append("")
+        out.append("Needs attention:")
+        for i, verdict, label in flagged:
+            out.append(f"  - #{i} [{verdict}] {label}")
+    return "\n".join(out)
+
+
+def _format_json(results: list) -> str:
+    summary: dict = {}
+    for r in results:
+        summary[r.verdict] = summary.get(r.verdict, 0) + 1
+    return json.dumps({
+        "count": len(results),
+        "summary": summary,
+        "results": [r.to_dict() for r in results],
+    }, default=str, indent=2)
+
+
+def _exit_code(results: list) -> int:
+    return 1 if any(r.verdict in ("not_found", "retracted") for r in results) else 0
+
+
+def main(argv: Optional[list] = None,
+         verify_fn: Callable = verify_reference) -> int:
+    parser = argparse.ArgumentParser(prog="matilde", description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("path", nargs="?", help="Path to a .bib file or a DOI list (one per line).")
+    parser.add_argument("--doi", help="Verify a single DOI instead of a file.")
+    parser.add_argument("--json", action="store_true", help="Emit JSON instead of text.")
+    parser.add_argument("--email", help="Contact email for provider polite pools (sets MATILDE_CONTACT_EMAIL).")
+    args = parser.parse_args(argv)
+
+    if args.email:
+        os.environ["MATILDE_CONTACT_EMAIL"] = args.email
+
+    if args.doi:
+        refs = [Reference(doi=args.doi)]
+    elif args.path:
+        try:
+            with open(args.path, encoding="utf-8") as fh:
+                text = fh.read()
+        except OSError as exc:
+            parser.error(f"cannot read {args.path}: {exc}")
+            return 2
+        refs = load_references(text, hint=args.path)
+    else:
+        parser.error("provide a file path or --doi")
+        return 2
+
+    if not refs:
+        print("No references found to verify.", file=sys.stderr)
+        return 2
+
+    results = [verify_fn(ref) for ref in refs]
+    print(_format_json(results) if args.json else _format_text(results))
+    return _exit_code(results)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

engine/parsing.py

@@ -0,0 +1,145 @@
+"""Bibliography parsing — turn BibTeX and loose DOI lists into ``Reference``s.
+
+Stdlib-only (no ``bibtexparser`` dependency) so the engine stays import-light. The
+BibTeX parser is brace-balanced and handles the common field shapes
+(``field = {value}`` / ``"value"`` / bareword) and the ``A and B and C`` author
+convention. It is not a full BibTeX grammar — it targets real-world reference
+lists, not every edge of the format.
+"""
+from __future__ import annotations
+
+import re
+
+from engine.citations import Reference, _normalize_doi
+
+
+def _strip_value(raw: str) -> str:
+    """Strip surrounding {}/"" delimiters, remove nested braces, collapse space."""
+    s = raw.strip().rstrip(",").strip()
+    if s and s[0] == '"' and s[-1] == '"':
+        s = s[1:-1]
+    s = s.replace("{", "").replace("}", "")
+    return " ".join(s.split())
+
+
+def _split_authors(value: str) -> list:
+    """Split a BibTeX author value on the ' and ' separator."""
+    parts = re.split(r"\s+and\s+", value.strip())
+    return [p.strip() for p in parts if p.strip()]
+
+
+def _iter_entry_bodies(text: str):
+    """Yield (entry_type, body) for each @type{...} block, brace-balanced."""
+    i, n = 0, len(text)
+    while i < n:
+        at = text.find("@", i)
+        if at == -1:
+            return
+        brace = text.find("{", at)
+        if brace == -1:
+            return
+        entry_type = text[at + 1:brace].strip().lower()
+        # balance braces from `brace`
+        depth, j = 0, brace
+        while j < n:
+            if text[j] == "{":
+                depth += 1
+            elif text[j] == "}":
+                depth -= 1
+                if depth == 0:
+                    break
+            j += 1
+        body = text[brace + 1:j]
+        yield entry_type, body
+        i = j + 1
+
+
+def _parse_fields(body: str) -> dict:
+    """Parse ``name = value`` fields from an entry body (skips the citekey)."""
+    # Drop the citekey (everything up to the first comma).
+    comma = body.find(",")
+    fields_blob = body[comma + 1:] if comma != -1 else body
+
+    fields: dict = {}
+    # Match: key = {balanced} | "quoted" | bareword , at top level.
+    pos, n = 0, len(fields_blob)
+    key_re = re.compile(r"\s*([A-Za-z][A-Za-z0-9_-]*)\s*=\s*")
+    while pos < n:
+        m = key_re.match(fields_blob, pos)
+        if not m:
+            pos += 1
+            continue
+        key = m.group(1).lower()
+        vstart = m.end()
+        if vstart >= n:
+            break
+        ch = fields_blob[vstart]
+        if ch == "{":
+            depth, j = 0, vstart
+            while j < n:
+                if fields_blob[j] == "{":
+                    depth += 1
+                elif fields_blob[j] == "}":
+                    depth -= 1
+                    if depth == 0:
+                        break
+                j += 1
+            value = fields_blob[vstart:j + 1]
+            pos = j + 1
+        elif ch == '"':
+            j = vstart + 1
+            while j < n and fields_blob[j] != '"':
+                j += 1
+            value = fields_blob[vstart:j + 1]
+            pos = j + 1
+        else:
+            j = vstart
+            while j < n and fields_blob[j] != ",":
+                j += 1
+            value = fields_blob[vstart:j]
+            pos = j + 1
+        fields[key] = _strip_value(value)
+    return fields
+
+
+def parse_bibtex(text: str) -> list:
+    """Parse a BibTeX string into a list of :class:`Reference`."""
+    refs = []
+    for entry_type, body in _iter_entry_bodies(text or ""):
+        if entry_type in ("comment", "string", "preamble"):
+            continue
+        f = _parse_fields(body)
+        if not f:
+            continue
+        year = None
+        if f.get("year"):
+            m = re.search(r"\d{4}", f["year"])
+            if m:
+                year = int(m.group(0))
+        refs.append(Reference(
+            raw=body.strip(),
+            title=f.get("title", ""),
+            authors=_split_authors(f["author"]) if f.get("author") else [],
+            year=year,
+            doi=f.get("doi", ""),
+            venue=f.get("journal") or f.get("booktitle") or "",
+            url=f.get("url", ""),
+        ))
+    return refs
+
+
+def parse_dois(text: str) -> list:
+    """Parse a newline-separated list of DOIs (bare, ``doi:`` or doi.org URLs).
+
+    Lines that are blank or start with ``#`` are ignored, as are lines that don't
+    contain a DOI-shaped token.
+    """
+    refs = []
+    for line in (text or "").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        doi = _normalize_doi(line)
+        if re.search(r"10\.\d{4,9}/", doi):
+            refs.append(Reference(doi=doi))
+    return refs

tests/test_cli.py

@@ -0,0 +1,93 @@
+"""Offline tests for the Matilde CLI — argument handling, formatting, exit codes.
+
+A fake ``verify_fn`` stands in for the network verifier so these run offline.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), "..")))
+
+from engine import cli  # noqa: E402
+from engine.citations import AxisResult, Reference, VerificationResult  # noqa: E402
+
+
+def _result(ref: Reference, verdict: str = "verified", score: float = 1.0) -> VerificationResult:
+    a = AxisResult(status="pass")
+    return VerificationResult(reference=ref, existence=a, metadata_match=a,
+                              retraction=a, url_liveness=a, score=score, verdict=verdict)
+
+
+def _verifier(verdict_for):
+    """Return a verify_fn that picks a verdict per reference via verdict_for(ref)."""
+    def _fn(ref):
+        return _result(ref, *verdict_for(ref))
+    return _fn
+
+
+def test_load_references_detects_bibtex():
+    refs = cli.load_references("@article{x, title={A}, doi={10.1/x}}")
+    assert len(refs) == 1 and refs[0].doi == "10.1/x"
+
+
+def test_load_references_falls_back_to_doi_list():
+    refs = cli.load_references("10.1038/171737a0\n10.1234/abcd")
+    assert [r.doi for r in refs] == ["10.1038/171737a0", "10.1234/abcd"]
+
+
+def test_main_single_doi_verified_exit_zero(capsys):
+    code = cli.main(["--doi", "10.1038/171737a0"],
+                    verify_fn=_verifier(lambda r: ("verified", 1.0)))
+    out = capsys.readouterr().out
+    assert code == 0
+    assert "verified" in out
+
+
+def test_main_flags_not_found_with_exit_one(tmp_path, capsys):
+    bib = tmp_path / "refs.bib"
+    bib.write_text("@article{a, title={Real}, doi={10.1/real}}\n"
+                   "@article{b, title={Fake}, doi={10.9/fake}}\n")
+
+    def verdict_for(ref):
+        return ("not_found", 0.1) if "fake" in ref.doi else ("verified", 1.0)
+
+    code = cli.main([str(bib)], verify_fn=_verifier(verdict_for))
+    out = capsys.readouterr().out
+    assert code == 1                       # a not_found ref fails the gate
+    assert "Needs attention" in out
+    assert "not_found" in out
+
+
+def test_main_json_output_is_valid(tmp_path, capsys):
+    bib = tmp_path / "refs.bib"
+    bib.write_text("@article{a, title={Real}, doi={10.1/real}}\n")
+    code = cli.main([str(bib), "--json"], verify_fn=_verifier(lambda r: ("verified", 1.0)))
+    payload = json.loads(capsys.readouterr().out)
+    assert code == 0
+    assert payload["count"] == 1
+    assert payload["summary"]["verified"] == 1
+
+
+def test_main_email_sets_env(tmp_path):
+    bib = tmp_path / "r.bib"
+    bib.write_text("@article{a, doi={10.1/x}}\n")
+    cli.main([str(bib), "--email", "researcher@uni.edu"],
+             verify_fn=_verifier(lambda r: ("verified", 1.0)))
+    assert os.environ.get("MATILDE_CONTACT_EMAIL") == "researcher@uni.edu"
+
+
+def test_main_no_input_is_usage_error():
+    with pytest.raises(SystemExit) as exc:
+        cli.main([], verify_fn=_verifier(lambda r: ("verified", 1.0)))
+    assert exc.value.code == 2
+
+
+def test_main_empty_file_returns_two(tmp_path):
+    empty = tmp_path / "empty.txt"
+    empty.write_text("# just a comment\n")
+    code = cli.main([str(empty)], verify_fn=_verifier(lambda r: ("verified", 1.0)))
+    assert code == 2