diff --git a/README.md b/README.md index 09feb02..8e0dcb6 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,13 @@ if result.is_valid: # no Error-level diagnostics frontmatter cannot be read — the `frontmatter-invalid-yaml` rule, which `validate_document`, given a parsed document, cannot report. +The public API is what the `legaldown` and `legaldown.validator` packages export (their +`__all__`). Changes to it are listed in the notes of each +[GitHub release](https://github.com/ForLegalAI/legaldown-validator/releases); before 1.0 a +minor release may change it, a patch release does not. Other modules are internal and may +change in any release; constants still only there are to be made public +([#34](https://github.com/ForLegalAI/legaldown-validator/issues/34)). + ### Working with the result Validating a document builds the indices the checks need — section numbers, resolved definitions, @@ -206,6 +213,19 @@ so a renderer or a UI can reuse the work instead of re-deriving it: | `sections`, `section_lookup` | Numbered section index; resolves `{{ref:}}` targets. Numbers count from the shallowest heading level, and a level a heading skips counts as 1 (`#`, `###`, `##` → 1, 1.1.1, 1.2), so no two sections share a number except alternatives and what they contain (§15.8) | | `definition_lookup`, `party_lookup`, `side_lookup`, `attachment_lookup` | Resolved display text | | `inline_dates`, `inline_money`, `inline_durations`, `inline_fields`, `inline_placeholders` | Field-spec values found in the body | +| `is_template` | Whether the document is a template (§15.1): it declares `questions`, carries a condition, or holds a `{{choose:}}` | +| `placed_markers` | The markers in body text that apply (§5.7, §15.3), in document order: `PlacedMarker(section, block, fragment, offset, source, identifier, condition, field, item, include_only, line)` — in fragment `fragment` of `block_fragments(block)`, at `offset`, which is the block's `field` (`text`, or `suffix` after a lifted `{{ref:}}`/`{{term:}}`); `item` is the list item it marks, counted in pre-order over all the list's items, nested and empty ones included, as `list_fragments` counts them; `identifier` is `""` where it does not apply (an include-only paragraph, §12.2). Identifiers and conditions are as written: check `is_valid` before relying on them | + +A renderer builds from these decisions rather than re-deriving them, with the helpers the +validator reads the document with: + +| Helper | What it gives | +|---|---| +| `lex(text)` → `Lexed` | The directives in inline text (§11.4), and a `view` of it with comments and code spans blanked; `is_escaped(text, offset)` | +| `block_fragments(block)`, `list_fragments(block)`, `list_items(block)` | The texts of a block that hold directives and markers, in the order `PlacedMarker.fragment` counts them (`Fragment(text, anchor)`); the same for a list, with the items each is in (`ListFragment(text, anchor, items)`), numbered in pre-order: an item before the items nested in it; a list's items, as `ListItem`s | +| `is_template(document)`, `is_drafting_note(block)` | The template decision without validating (§15.1); whether a quote block is a drafting note (§15.6) | +| `legaldown.validator`: `parse_condition` → `Condition`, `condition_problem`, `exclusive`, `Presence`, `ALWAYS` | Conditions (§15.3, §15.4): parse one, tell why one is invalid, tell whether two units (each the set of conditions it appears under, `Presence`) can never appear together, given the document's `questions` | +| `legaldown.validator`: `is_valid_iso_date`, `is_valid_money_amount`, `is_positive_numeric`, `IDENTIFIER_RE`, `KNOWN_CURRENCIES` | Value checks (§3.10, §10) | ### Reading and editing the document model diff --git a/src/legaldown/__init__.py b/src/legaldown/__init__.py index 519913b..86b4c53 100644 --- a/src/legaldown/__init__.py +++ b/src/legaldown/__init__.py @@ -33,16 +33,23 @@ DELIMITER_PAIRS, DefinitionAnchor, DefinitionRef, + Fragment, + ListFragment, + block_fragments, collect_definitions, definition_lookup, find_definition_anchors, id_term, + list_fragments, ) from .directives import ( DIRECTIVE_PARAMS, KNOWN_DIRECTIVES, Directive, + Lexed, + is_escaped, iter_directives, + lex, ) from .models import ( BLOCK_DEFAULTS, @@ -62,6 +69,7 @@ document_to_dict, empty_document, item_text, + list_items, metadata_from_dict, party_from_dict, section_from_dict, @@ -80,8 +88,11 @@ AttachmentDefinitionsImporter, DefinitionsImporter, Diagnostic, + PlacedMarker, SectionIndexEntry, ValidationResult, + is_drafting_note, + is_template, slugify_identifier, validate_document, ) @@ -126,8 +137,15 @@ "DELIMITER_PAIRS", "find_definition_anchors", "DefinitionAnchor", - # Directives (§11) + # Directives (§11): the lexer, and where text holding them is "iter_directives", + "lex", + "Lexed", + "is_escaped", + "block_fragments", + "list_fragments", + "Fragment", + "ListFragment", "Directive", "DIRECTIVE_PARAMS", "KNOWN_DIRECTIVES", @@ -138,6 +156,10 @@ "ValidationResult", "SectionIndexEntry", "Diagnostic", + "PlacedMarker", + # What validate_document decides, on its own (§15.1, §15.6) + "is_template", + "is_drafting_note", # Document model "Document", "Metadata", @@ -145,6 +167,7 @@ "Block", "ListItem", "item_text", + "list_items", "Side", "Party", "Representative", diff --git a/src/legaldown/definitions.py b/src/legaldown/definitions.py index 584e8d8..46084a8 100644 --- a/src/legaldown/definitions.py +++ b/src/legaldown/definitions.py @@ -15,6 +15,7 @@ from collections.abc import Callable, Iterator from dataclasses import dataclass +from typing import NamedTuple from .directives import Directive, Lexed, blank_fenced_code, lex, mask_directives from .markdown import FENCE_OPEN_RE, is_indented_code @@ -177,7 +178,25 @@ class DefinitionRef: offset: int = 0 -def block_fragments(block: Block) -> list[tuple[str, bool]]: +class Fragment(NamedTuple): + """A text of a block that may hold directives and markers + (``block_fragments``): the *text*, and whether its end is an anchor + position (§5.7).""" + + text: str + anchor: bool + + +class ListFragment(NamedTuple): + """A ``Fragment`` of a list (``list_fragments``), with the *items* it is + in: each item's number, outermost first.""" + + text: str + anchor: bool + items: tuple[int, ...] + + +def block_fragments(block: Block) -> list[Fragment]: """The free-text fragments of *block* that may contain inline directives, each with whether a ``{#id}`` at its very end is in an anchor position. @@ -190,17 +209,18 @@ def block_fragments(block: Block) -> list[tuple[str, bool]]: code span or comment left open — runs into the next, and code and raw HTML in it hold none (§11.4). """ - return [(text, position) for text, position, _items in _fragments(block)] + return [Fragment(text, position) for text, position, _items in _fragments(block)] -def list_fragments(block: Block) -> list[tuple[str, bool, tuple[int, ...]]]: +def list_fragments(block: Block) -> list[ListFragment]: """The fragments of a list's items, as ``block_fragments`` lists them: each with whether its end is an anchor position — the end of an item's first paragraph, the block it opens with (§5.7) — and the items it is - in, outermost first, each item numbered in document order among all the - list's items, nested ones included (§15.3). A list in a block quote is - no unit: its fragments are in the items the quote is in.""" - return _fragments(block) + in, outermost first. Items are numbered in pre-order among all the + list's items: an item before the items nested in it, empty items + included (§15.3). A list in a block quote is no unit: its fragments are + in the items the quote is in.""" + return [ListFragment(*fragment) for fragment in _fragments(block)] def quote_ranges(block: Block) -> list[tuple[str, range]]: diff --git a/src/legaldown/directives.py b/src/legaldown/directives.py index 7006456..54780e6 100644 --- a/src/legaldown/directives.py +++ b/src/legaldown/directives.py @@ -315,9 +315,10 @@ def lex(text: str) -> Lexed: """Lex *text* for directives, outside literal regions (§11.4). *text* is inline text, such as a paragraph's: fenced code is not looked - for. Text that can hold one (a code block's, a block quote's) is passed - through ``blank_fenced_code`` first, as block structure precedes inline - structure. It is read once, left to right, taking whichever + for. Text that can hold it (a code block's, a block quote's) has it + blanked first, as block structure precedes inline structure + (``blank_fenced_code``; ``block_fragments`` gives such text so). It is + read once, left to right, taking whichever of a directive, a comment, or a code span opens first. A directive is lexed from the source as written and consumes its own text, so a quoted value may hold backticks or ``\n") + assert marker.offset == 5 + + +@pytest.mark.parametrize("body", [ + "# A\n\n| a |\n|---|\n| x {#t} |\n", + "# A\n\n> quoted {#t}\n", + "# A\n\n> > nested {#t}\n", + "# A\n\nText {#t} more.\n", + "# A\n\n- a\n\n second {#t}\n", +]) +def test_a_marker_out_of_place_is_not_placed(body): + assert _markers(body) == [] + + +def test_a_document_built_in_code_has_no_lines(): + document = document_from_dict({"sections": [{"title": "A", "blocks": [{"kind": "paragraph", "text": "Text {#t}"}]}]}) + [marker] = validate_document(document).placed_markers + assert (marker.identifier, marker.line) == ("t", None) + + +def test_is_template_on_its_own(): + assert not is_template(parse_document(_FRONTMATTER + "# A\n\nText.\n")) + assert is_template(parse_document(_FRONTMATTER + "# A\n\nPick {{choose: q}}.\n")) + + +def test_is_drafting_note(): + [note, quote] = parse_document(_FRONTMATTER + "# A\n\n> [!drafting]\n> Check.\n\n> Plain.\n").sections[0].blocks + assert is_drafting_note(note) and not is_drafting_note(quote) + + +def _corpus(): + root = os.environ.get("LEGALDOWN_FIXTURES_DIR", "") + if not root or not Path(root).is_dir(): + return [] + spec = Path(root).parent + return [pytest.param(path, id=path.relative_to(spec).as_posix()) for path in sorted(spec.rglob("*.lgd"))] + + +@pytest.mark.parametrize("path", _corpus()) +def test_the_decisions_are_the_validators_own(path: Path): + """Over the specification's documents: the template decision is + ``is_template``, the placed markers are the validator's placed findings, + each where it says, at most one in a fragment.""" + try: + document = parse_document(path.read_text(encoding="utf-8")) + except legaldown.FrontmatterError: + pytest.skip("unreadable frontmatter") + result = validate_document(document) + assert result.is_template == is_template(document) + found = [f for f in find_markers(document, lex) if f.marker is not None and f.placed(result.is_template)] + assert [(m.section, m.block, m.fragment, m.offset, m.source) for m in result.placed_markers] == [ + (f.section, f.block, f.fragment, f.offset, f.source) for f in found + ] + places = [(m.section, m.block, m.fragment) for m in result.placed_markers] + assert len(places) == len(set(places)) + lines = path.read_text(encoding="utf-8").split("\n") + for marker in result.placed_markers: + blocks = document.preamble if marker.section is None else document.sections[marker.section].blocks + text = block_fragments(blocks[marker.block])[marker.fragment][0] + assert text[marker.offset:marker.offset + len(marker.source)] == marker.source + assert marker.source in lines[marker.line - 1] + + +def test_a_marker_after_a_lifted_reference_is_in_the_suffix(): + [marker] = _markers("# A\n\nSee {{ref: a}} below. {#t}\n\n# B {#a}\n\nText.\n") + block = parse_document(_FRONTMATTER + "# A\n\nSee {{ref: a}} below. {#t}\n\n# B {#a}\n\nText.\n").sections[0].blocks[0] + assert (block.kind, marker.field) == ("ref", "suffix") + assert block.suffix[marker.offset:] == "{#t}" + + +def test_a_list_items_marker_has_its_line(): + body = "# A\n\n- first {#a}\n\n second para\n- \n- third\n more {#c}\n" + markers = _markers(body) + lines = (_FRONTMATTER + body).split("\n") + assert [(m.identifier, m.item) for m in markers] == [("a", 0), ("c", 2)] + assert [lines[m.line - 1] for m in markers] == ["- first {#a}", " more {#c}"] + + +def test_a_list_is_walked_once(monkeypatch): + """Its fragments are read once for the placed markers, however many it + holds (not once a marker, which is quadratic in a long list).""" + import legaldown.definitions + + calls = [] + real = legaldown.definitions.list_fragments + monkeypatch.setattr(legaldown.definitions, "list_fragments", lambda block: calls.append(1) or real(block)) + markers = _markers("# A\n\n" + "".join(f"- item {{#i{n}}}\n" for n in range(50))) + assert [m.item for m in markers] == list(range(50)) + assert len(calls) <= 2 # the units' reading, and the placed markers' + + +def test_a_document_changed_since_parsing_has_no_lines(): + document = parse_document(_FRONTMATTER + "# A\n\nText {#t}\n\n- a {#b}\n") + document.sections[0].blocks.insert(0, document.sections[0].blocks[0]) + assert [m.line for m in validate_document(document).placed_markers] == [None, None, None] + + +def test_a_list_of_string_items_built_in_code(): + document = document_from_dict({"sections": [{"title": "A", "blocks": [ + {"kind": "unordered_list", "items": ["a {#x}", "b\n\n - c {#y}"]}, + ]}]}) + assert [(m.identifier, m.item) for m in validate_document(document).placed_markers] == [("x", 0), ("y", 2)] + + +def test_two_lists_in_a_section_number_their_own_items(): + """Each list's fragments are its own: the first list's fragment 1 is in + its item 0, the second's in its item 1.""" + markers = _markers("# A\n\n- a {#a}\n\n more\n- b {#b}\n\nText.\n\n1. c\n2. d {#d}\n") + assert [(m.identifier, m.block, m.fragment, m.item) for m in markers] == [ + ("a", 0, 0, 0), ("b", 0, 2, 1), ("d", 2, 1, 1), + ] + + +def test_a_marker_after_a_lifted_term_is_in_the_suffix(): + body = '"Thing" {{def: thing}} means a thing.\n\n# A\n\nUse the {{term: thing}} well. {#t}\n' + [marker] = _markers(body) + block = parse_document(_FRONTMATTER + body).sections[0].blocks[0] + assert (block.kind, marker.field) == ("term", "suffix") + assert block.suffix[marker.offset:] == "{#t}" + + +def test_only_a_quote_block_is_a_drafting_note(): + from legaldown import Block + + assert not is_drafting_note(Block(kind="paragraph", text="[!DRAFTING]\nx")) + assert is_drafting_note(Block(kind="quote", text="[!DRAFTING]\nx")) + + +def test_fragments_have_names(): + [block] = parse_document(_FRONTMATTER + "# A\n\n- a {#a}\n - b\n").sections[0].blocks + first = list_fragments(block)[0] + assert (first.text, first.anchor, first.items) == ("a {#a}", True, (0,)) + text, anchor = block_fragments(block)[1] + assert (text, anchor) == ("b", True)