diff --git a/README.md b/README.md index c304d73..37df591 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,31 @@ result = verify_reference(ref) print(result.verdict, result.score) # -> e.g. "verified" 1.0 ``` +### Command line — verify a whole bibliography + +```bash +python3 -m engine.cli refs.bib # verify a BibTeX file +python3 -m engine.cli dois.txt # or a list of DOIs (one per line) +python3 -m engine.cli --doi 10.1038/171737a0 # or a single DOI +python3 -m engine.cli refs.bib --json # machine-readable +``` + +Output for a mixed bibliography: + +``` +Verified 3 reference(s): + [OK ] verified 0.9 Molecular structure of nucleic acids + [RET] retracted 0.1 Ileal-lymphoid-nodular hyperplasia + [XX ] not_found 0.1 A Totally Fabricated Paper + +not_found=1 retracted=1 verified=1 +``` + +Exit code is non-zero if any reference is `not_found` or `retracted` — so you can +drop it into CI or a pre-commit hook on a manuscript's `.bib`. + +### Tests + ```bash python3 -m pytest tests/ --ignore=tests/test_citations_integration.py # offline unit suite MATILDE_LIVE=1 python3 -m pytest tests/test_citations_integration.py # live API checks diff --git a/engine/cli.py b/engine/cli.py new file mode 100644 index 0000000..1c4a99f --- /dev/null +++ b/engine/cli.py @@ -0,0 +1,117 @@ +"""Matilde command-line interface — verify a bibliography from the terminal. + +Usage:: + + python3 -m engine.cli refs.bib # verify a BibTeX file + python3 -m engine.cli dois.txt # verify a list of DOIs + python3 -m engine.cli --doi 10.1038/171737a0 # verify one DOI + python3 -m engine.cli refs.bib --json # machine-readable output + python3 -m engine.cli refs.bib --email you@uni.edu # polite-pool contact + +Exit codes: 0 = all references verified / only warnings; 1 = at least one +``not_found`` or ``retracted`` reference (useful as a pre-commit / CI gate on a +manuscript's .bib); 2 = usage error. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Callable, Optional + +from engine.citations import Reference, verify_reference +from engine.parsing import parse_bibtex, parse_dois + +_VERDICT_MARK = { + "verified": "OK ", + "warnings": "! ", + "unverifiable": "? ", + "not_found": "XX ", + "retracted": "RET", +} + + +def load_references(text: str, hint: str = "") -> list: + """Load references from *text*, choosing BibTeX vs DOI-list by content/hint.""" + if "@" in text and "=" in text or hint.endswith(".bib"): + refs = parse_bibtex(text) + if refs: + return refs + return parse_dois(text) + + +def _format_text(results: list) -> str: + lines, summary = [], {} + flagged = [] + for i, r in enumerate(results): + summary[r.verdict] = summary.get(r.verdict, 0) + 1 + label = r.reference.title or r.reference.doi or r.reference.raw[:50] or "(no id)" + mark = _VERDICT_MARK.get(r.verdict, " ") + lines.append(f" [{mark}] {r.verdict:<12} {r.score:>4} {label}") + if r.verdict in ("not_found", "retracted"): + flagged.append((i, r.verdict, label)) + header = f"Verified {len(results)} reference(s):" + tally = " ".join(f"{k}={v}" for k, v in sorted(summary.items())) + out = [header, *lines, "", tally] + if flagged: + out.append("") + out.append("Needs attention:") + for i, verdict, label in flagged: + out.append(f" - #{i} [{verdict}] {label}") + return "\n".join(out) + + +def _format_json(results: list) -> str: + summary: dict = {} + for r in results: + summary[r.verdict] = summary.get(r.verdict, 0) + 1 + return json.dumps({ + "count": len(results), + "summary": summary, + "results": [r.to_dict() for r in results], + }, default=str, indent=2) + + +def _exit_code(results: list) -> int: + return 1 if any(r.verdict in ("not_found", "retracted") for r in results) else 0 + + +def main(argv: Optional[list] = None, + verify_fn: Callable = verify_reference) -> int: + parser = argparse.ArgumentParser(prog="matilde", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("path", nargs="?", help="Path to a .bib file or a DOI list (one per line).") + parser.add_argument("--doi", help="Verify a single DOI instead of a file.") + parser.add_argument("--json", action="store_true", help="Emit JSON instead of text.") + parser.add_argument("--email", help="Contact email for provider polite pools (sets MATILDE_CONTACT_EMAIL).") + args = parser.parse_args(argv) + + if args.email: + os.environ["MATILDE_CONTACT_EMAIL"] = args.email + + if args.doi: + refs = [Reference(doi=args.doi)] + elif args.path: + try: + with open(args.path, encoding="utf-8") as fh: + text = fh.read() + except OSError as exc: + parser.error(f"cannot read {args.path}: {exc}") + return 2 + refs = load_references(text, hint=args.path) + else: + parser.error("provide a file path or --doi") + return 2 + + if not refs: + print("No references found to verify.", file=sys.stderr) + return 2 + + results = [verify_fn(ref) for ref in refs] + print(_format_json(results) if args.json else _format_text(results)) + return _exit_code(results) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/engine/parsing.py b/engine/parsing.py new file mode 100644 index 0000000..d84f7a1 --- /dev/null +++ b/engine/parsing.py @@ -0,0 +1,145 @@ +"""Bibliography parsing — turn BibTeX and loose DOI lists into ``Reference``s. + +Stdlib-only (no ``bibtexparser`` dependency) so the engine stays import-light. The +BibTeX parser is brace-balanced and handles the common field shapes +(``field = {value}`` / ``"value"`` / bareword) and the ``A and B and C`` author +convention. It is not a full BibTeX grammar — it targets real-world reference +lists, not every edge of the format. +""" +from __future__ import annotations + +import re + +from engine.citations import Reference, _normalize_doi + + +def _strip_value(raw: str) -> str: + """Strip surrounding {}/"" delimiters, remove nested braces, collapse space.""" + s = raw.strip().rstrip(",").strip() + if s and s[0] == '"' and s[-1] == '"': + s = s[1:-1] + s = s.replace("{", "").replace("}", "") + return " ".join(s.split()) + + +def _split_authors(value: str) -> list: + """Split a BibTeX author value on the ' and ' separator.""" + parts = re.split(r"\s+and\s+", value.strip()) + return [p.strip() for p in parts if p.strip()] + + +def _iter_entry_bodies(text: str): + """Yield (entry_type, body) for each @type{...} block, brace-balanced.""" + i, n = 0, len(text) + while i < n: + at = text.find("@", i) + if at == -1: + return + brace = text.find("{", at) + if brace == -1: + return + entry_type = text[at + 1:brace].strip().lower() + # balance braces from `brace` + depth, j = 0, brace + while j < n: + if text[j] == "{": + depth += 1 + elif text[j] == "}": + depth -= 1 + if depth == 0: + break + j += 1 + body = text[brace + 1:j] + yield entry_type, body + i = j + 1 + + +def _parse_fields(body: str) -> dict: + """Parse ``name = value`` fields from an entry body (skips the citekey).""" + # Drop the citekey (everything up to the first comma). + comma = body.find(",") + fields_blob = body[comma + 1:] if comma != -1 else body + + fields: dict = {} + # Match: key = {balanced} | "quoted" | bareword , at top level. + pos, n = 0, len(fields_blob) + key_re = re.compile(r"\s*([A-Za-z][A-Za-z0-9_-]*)\s*=\s*") + while pos < n: + m = key_re.match(fields_blob, pos) + if not m: + pos += 1 + continue + key = m.group(1).lower() + vstart = m.end() + if vstart >= n: + break + ch = fields_blob[vstart] + if ch == "{": + depth, j = 0, vstart + while j < n: + if fields_blob[j] == "{": + depth += 1 + elif fields_blob[j] == "}": + depth -= 1 + if depth == 0: + break + j += 1 + value = fields_blob[vstart:j + 1] + pos = j + 1 + elif ch == '"': + j = vstart + 1 + while j < n and fields_blob[j] != '"': + j += 1 + value = fields_blob[vstart:j + 1] + pos = j + 1 + else: + j = vstart + while j < n and fields_blob[j] != ",": + j += 1 + value = fields_blob[vstart:j] + pos = j + 1 + fields[key] = _strip_value(value) + return fields + + +def parse_bibtex(text: str) -> list: + """Parse a BibTeX string into a list of :class:`Reference`.""" + refs = [] + for entry_type, body in _iter_entry_bodies(text or ""): + if entry_type in ("comment", "string", "preamble"): + continue + f = _parse_fields(body) + if not f: + continue + year = None + if f.get("year"): + m = re.search(r"\d{4}", f["year"]) + if m: + year = int(m.group(0)) + refs.append(Reference( + raw=body.strip(), + title=f.get("title", ""), + authors=_split_authors(f["author"]) if f.get("author") else [], + year=year, + doi=f.get("doi", ""), + venue=f.get("journal") or f.get("booktitle") or "", + url=f.get("url", ""), + )) + return refs + + +def parse_dois(text: str) -> list: + """Parse a newline-separated list of DOIs (bare, ``doi:`` or doi.org URLs). + + Lines that are blank or start with ``#`` are ignored, as are lines that don't + contain a DOI-shaped token. + """ + refs = [] + for line in (text or "").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + doi = _normalize_doi(line) + if re.search(r"10\.\d{4,9}/", doi): + refs.append(Reference(doi=doi)) + return refs diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..fdec291 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,93 @@ +"""Offline tests for the Matilde CLI — argument handling, formatting, exit codes. + +A fake ``verify_fn`` stands in for the network verifier so these run offline. +""" +from __future__ import annotations + +import json +import os +import sys + +import pytest + +sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) + +from engine import cli # noqa: E402 +from engine.citations import AxisResult, Reference, VerificationResult # noqa: E402 + + +def _result(ref: Reference, verdict: str = "verified", score: float = 1.0) -> VerificationResult: + a = AxisResult(status="pass") + return VerificationResult(reference=ref, existence=a, metadata_match=a, + retraction=a, url_liveness=a, score=score, verdict=verdict) + + +def _verifier(verdict_for): + """Return a verify_fn that picks a verdict per reference via verdict_for(ref).""" + def _fn(ref): + return _result(ref, *verdict_for(ref)) + return _fn + + +def test_load_references_detects_bibtex(): + refs = cli.load_references("@article{x, title={A}, doi={10.1/x}}") + assert len(refs) == 1 and refs[0].doi == "10.1/x" + + +def test_load_references_falls_back_to_doi_list(): + refs = cli.load_references("10.1038/171737a0\n10.1234/abcd") + assert [r.doi for r in refs] == ["10.1038/171737a0", "10.1234/abcd"] + + +def test_main_single_doi_verified_exit_zero(capsys): + code = cli.main(["--doi", "10.1038/171737a0"], + verify_fn=_verifier(lambda r: ("verified", 1.0))) + out = capsys.readouterr().out + assert code == 0 + assert "verified" in out + + +def test_main_flags_not_found_with_exit_one(tmp_path, capsys): + bib = tmp_path / "refs.bib" + bib.write_text("@article{a, title={Real}, doi={10.1/real}}\n" + "@article{b, title={Fake}, doi={10.9/fake}}\n") + + def verdict_for(ref): + return ("not_found", 0.1) if "fake" in ref.doi else ("verified", 1.0) + + code = cli.main([str(bib)], verify_fn=_verifier(verdict_for)) + out = capsys.readouterr().out + assert code == 1 # a not_found ref fails the gate + assert "Needs attention" in out + assert "not_found" in out + + +def test_main_json_output_is_valid(tmp_path, capsys): + bib = tmp_path / "refs.bib" + bib.write_text("@article{a, title={Real}, doi={10.1/real}}\n") + code = cli.main([str(bib), "--json"], verify_fn=_verifier(lambda r: ("verified", 1.0))) + payload = json.loads(capsys.readouterr().out) + assert code == 0 + assert payload["count"] == 1 + assert payload["summary"]["verified"] == 1 + + +def test_main_email_sets_env(tmp_path): + bib = tmp_path / "r.bib" + bib.write_text("@article{a, doi={10.1/x}}\n") + cli.main([str(bib), "--email", "researcher@uni.edu"], + verify_fn=_verifier(lambda r: ("verified", 1.0))) + assert os.environ.get("MATILDE_CONTACT_EMAIL") == "researcher@uni.edu" + + +def test_main_no_input_is_usage_error(): + with pytest.raises(SystemExit) as exc: + cli.main([], verify_fn=_verifier(lambda r: ("verified", 1.0))) + assert exc.value.code == 2 + + +def test_main_empty_file_returns_two(tmp_path): + empty = tmp_path / "empty.txt" + empty.write_text("# just a comment\n") + code = cli.main([str(empty)], verify_fn=_verifier(lambda r: ("verified", 1.0))) + assert code == 2 diff --git a/tests/test_parsing.py b/tests/test_parsing.py new file mode 100644 index 0000000..cbe78ca --- /dev/null +++ b/tests/test_parsing.py @@ -0,0 +1,69 @@ +"""Tests for bibliography parsing — BibTeX and loose DOI lists into References.""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) + +from engine.parsing import parse_bibtex, parse_dois # noqa: E402 + + +SAMPLE_BIB = r""" +@article{vaswani2017attention, + title = {Attention Is All You Need}, + author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki}, + year = {2017}, + journal = {Advances in Neural Information Processing Systems}, + doi = {10.48550/arXiv.1706.03762}, + url = {https://arxiv.org/abs/1706.03762} +} + +@inproceedings{he2016deep, + title = "Deep Residual Learning for Image Recognition", + author = "He, Kaiming and Zhang, Xiangyu", + year = 2016 +} +""" + + +def test_parse_bibtex_extracts_fields(): + refs = parse_bibtex(SAMPLE_BIB) + assert len(refs) == 2 + r0 = refs[0] + assert r0.title == "Attention Is All You Need" + assert r0.year == 2017 + assert r0.doi == "10.48550/arXiv.1706.03762" + assert r0.url == "https://arxiv.org/abs/1706.03762" + assert r0.authors == ["Vaswani, Ashish", "Shazeer, Noam", "Parmar, Niki"] + + +def test_parse_bibtex_handles_quoted_values_and_bare_year(): + refs = parse_bibtex(SAMPLE_BIB) + r1 = refs[1] + assert r1.title == "Deep Residual Learning for Image Recognition" + assert r1.year == 2016 + assert r1.authors == ["He, Kaiming", "Zhang, Xiangyu"] + assert r1.doi == "" # no doi present + + +def test_parse_bibtex_empty_string_returns_empty(): + assert parse_bibtex("") == [] + assert parse_bibtex("no entries here") == [] + + +def test_parse_bibtex_strips_nested_braces_in_title(): + bib = r"@article{x, title = {The {BERT} Model}, year = {2019}}" + refs = parse_bibtex(bib) + assert refs[0].title == "The BERT Model" + + +def test_parse_dois_one_per_line(): + text = """ + 10.1038/171737a0 + https://doi.org/10.1016/j.cell.2020.01.001 + doi:10.1234/abcd + # a comment line, ignored + """ + refs = parse_dois(text) + assert [r.doi for r in refs] == ["10.1038/171737a0", "10.1016/j.cell.2020.01.001", "10.1234/abcd"]