From 4ecf0d3502cb2668ced1201073cda4dd4d0d33e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 06:54:23 +0000 Subject: [PATCH] 0.4.0: top-level legaldown is the workflow and the model; Document.from_dict/to_dict; Template(text, resolve=) legaldown.__all__ goes from 81 names to 48: the workflow (load, parse, save, serialize, validate, the template and answers calls), the result types (now with Blank), is_template, the document model, and the deprecated names until 0.5.0. The lexer, fragment and definition helpers, render_block/render_item, the dict factories and BLOCK_DEFAULTS leave __all__ but still import from legaldown as the same objects, without a warning, so PactTrack and legaldown-render keep working; their supported home is legaldown.syntax / legaldown.grammar, which gain the definition readers (collect_definitions, definition_lookup, id_term, DefinitionRef, find_definition_anchors, DefinitionAnchor) and render_block/render_item. Document.from_dict(data) and document.to_dict() are the documented way to build and dump a document; Document() is a blank document (empty_document() is a starter contract, now said so). Template(text, *, resolve=None): path is set by load_template (still a readable attribute), and check= becomes private (_check), used only by tests and the conformance harness. Tests pin the new __all__ and `import *`, and check every name PactTrack and legaldown-render import from legaldown still imports, warning-free, as the same object as its new home. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013WmBAc5T7UCKVdpUmg9qxz --- README.md | 58 +++++-- src/legaldown/__init__.py | 120 +++++---------- src/legaldown/models.py | 36 ++++- src/legaldown/serializer.py | 2 +- src/legaldown/syntax.py | 32 +++- src/legaldown/template.py | 22 ++- tests/conformance/test_legaldown_fixtures.py | 2 +- tests/test_assembly.py | 2 +- tests/test_public_api.py | 154 +++++++++++++++++-- tests/test_tooling_surface.py | 37 ++++- 10 files changed, 338 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index f102925..50caf76 100644 --- a/README.md +++ b/README.md @@ -277,11 +277,18 @@ is the same function for files on disk, and the one `parse_template` takes as `r > `used_terms`, which only the validator used. The public API is what the `legaldown` and `legaldown.validator` packages export (their -`__all__`). Changes to it are listed in the notes of each +`__all__`), and `legaldown.grammar` and `legaldown.syntax`. `legaldown` is the workflow and the +model: loading, validating, saving and assembling documents, what that returns, and the `Document` +dataclasses. `legaldown.syntax` and `legaldown.grammar` are the tooling API (see +[Tooling API](#tooling-api)): reading source text the way the validator does, and the language's +constants and rules. The names that moved out of `legaldown.__all__` in 0.4.0 — the lexer, the +fragments, the definition readers, `render_block`, the dict factories and the like — still import +from `legaldown`, as the same objects and without a warning, so code written for earlier releases +keeps working; new code imports them from their home (the table at the end of +[Tooling API](#tooling-api)). Changes to the API are listed in the notes of each [GitHub release](https://github.com/ForLegalAI/legaldown-validator/releases); before 1.0 a minor release may change it, a patch release does not. Other modules are internal and may -change in any release, except `legaldown.grammar` and `legaldown.syntax` (see -[Tooling API](#tooling-api)), which are public too. +change in any release. ### Working with the result @@ -305,7 +312,7 @@ itself is what was found, kept nowhere but in `diagnostics`: | `values` | The field-spec values the checks met, as written, as `InlineValues`: `dates`, `money`, `durations`, `fields`, `placeholders` (those of the frontmatter too; one with malformed arguments is not among them) | | `blanks` | The document's blanks, a template's or not, by placeholder id in order of first occurrence: `Blank(id, type, fixed, in_frontmatter, consistent)` — `type` the type the validator settled on: the effective type of the first occurrence with a valid one (§10.7, §15.2; `text` when none is written), or `None` when no occurrence has a usable type (each is written with an invalid one, or the id is a decision question's); `fixed` the currency of a money blank or the unit of a duration blank that every occurrence fixes (`""` when one fixes none, or the blank is not `consistent`); `consistent` false when its occurrences have different types or fix two currencies or units (placeholder-type-inconsistent); `in_frontmatter` whether one occurrence is in the frontmatter (§3.10) | | `is_template` | Whether the document is a template (§15.1): it declares `questions`, carries a condition, or holds a `{{choose:}}` | -| `placed_markers` | The markers in body text that apply (§5.7, §15.3), in document order: `PlacedMarker(section, block, fragment, offset, source, identifier, condition, field, item, include_only, line)` — in fragment `fragment` of `block_fragments(block)`, at `offset`, which is the block's `field` (`text`, or `suffix` after a lifted `{{ref:}}`/`{{term:}}`); `item` is the list item it marks, counted in pre-order over all the list's items, nested and empty ones included, as `list_fragments` counts them; `identifier` is `""` where it does not apply (an include-only paragraph, §12.2). Identifiers and conditions are as written: check `is_valid` before relying on them | +| `placed_markers` | The markers in body text that apply (§5.7, §15.3), in document order: `PlacedMarker(section, block, fragment, offset, source, identifier, condition, field, item, include_only, line)` — in fragment `fragment` of `block_fragments(block)` (`legaldown.syntax`), at `offset`, which is the block's `field` (`text`, or `suffix` after a lifted `{{ref:}}`/`{{term:}}`); `item` is the list item it marks, counted in pre-order over all the list's items, nested and empty ones included, as `list_fragments` counts them; `identifier` is `""` where it does not apply (an include-only paragraph, §12.2). Identifiers and conditions are as written: check `is_valid` before relying on them | `is_template(document)` (top-level) gives the template decision without validating (§15.1). A renderer builds from these decisions rather than re-deriving them, reading the source with @@ -345,9 +352,11 @@ A `Section`'s `identifier` is the explicit `{#id}` written after its heading, or it has none. The identifiers the validator generates (§5.3, §5.5) are not written back into the model: read them from `result.index.sections`. -`document_to_dict()` / `document_from_dict()` round-trip the model through JSON-friendly -structures, except the fields that describe the parsed source rather than the document, -`Metadata.not_line_editable`, `Metadata.frontmatter_absent` and `Document.source_map`. +`document.to_dict()` and `Document.from_dict(data)` round-trip the model through JSON-friendly +structures (missing fields take their defaults), except the fields that describe the parsed source +rather than the document, `Metadata.not_line_editable`, `Metadata.frontmatter_absent`, +`Document.source_map` and `Document.path`. A blank document is `Document()`; `empty_document()`, +still importable from `legaldown`, is not one but a starter contract with two sides and two sections. Each diagnostic names its `line` (from 1) and its `file` (the document's `filename`), as §16.9 requires: the line of the directive, marker, heading or block it is about, or of the @@ -355,11 +364,11 @@ frontmatter key — for a missing key, the key that holds it, or the frontmatter come from the `source_map` that `load` and `parse` give a document. A document built from a dict has none, and one changed after parsing no longer fits its map: their diagnostics have no line (`None`) rather than a stale one. `FrontmatterError.line` is the line of YAML that cannot be -read. `render_block()` renders a single block when you are driving your own layout. -`iter_directives()` lexes the directives in a piece of text by the §11.2 +read. `render_block()` (`legaldown.syntax`) renders a single block when you are driving your own +layout. `iter_directives()` (`legaldown.syntax`) lexes the directives in a piece of text by the §11.2 grammar — parameters in any order, quoted values decoded — and is what the validator itself uses. -`definition_lookup(collect_definitions(document))` gives a document's definitions, id to term, +`definition_lookup(collect_definitions(document))` (`legaldown.syntax`) gives a document's definitions, id to term, without validating it: the same map as `result.index.definition_lookup`, except for definitions imported from an amended original or an attachment file. A term written empty (`"" {{def: x}}`) reads as its id (`id_term`), and an id that is not a valid identifier is left out. @@ -392,8 +401,9 @@ kind `paragraph`: a directive in it stays in its text. A heading in an item or a `heading` block, not a section: its `text` without the `#` syntax, and its `level` (1–6; `level` is 0 on every other block). Written outside an item, where it would be a section's heading, a `heading` block becomes paragraph text with a backslash before it. `item_text(item)` gives an -item's first paragraph's text, and `render_item(item)` its content as written after its marker; -a string item in `document_from_dict` (as models before 0.3 held them) is read as such content. +item's first paragraph's text, and `render_item(item)` its content as written after its marker +(both in `legaldown.syntax`); a string item in `Document.from_dict` (as models before 0.3 held +them) is read as such content. A nested item's condition applies within those of the items it is nested in (§15.3). A quote block keeps its `text`: its content, one line per source line without the `>` marker. @@ -439,9 +449,10 @@ the document. Tools built on the validator (renderers, editors, importers) need the language itself: its vocabulary, its value formats, and how the validator reads source text. Two modules hand that over. They are part of the supported API and covered by the same versioning as `legaldown` -and `legaldown.validator`, and they are the only supported way to reach these names: every +and `legaldown.validator`, and they are the supported way to reach these names: every module path below them (`legaldown.markdown`, `legaldown.validator.patterns`, …) is internal -and may be reorganised in any release. Their names are the validator's own objects, not copies, so +and may be reorganised in any release (some of the names also still import from `legaldown`, +for compatibility). Their names are the validator's own objects, not copies, so a tool and the validator cannot disagree. **`legaldown.grammar`** is what needs no document: constants and small rules. @@ -491,6 +502,9 @@ for _section, _index, block in document.iter_blocks(): | `Marker`, `MARKER_RE`, `parse_marker`, `format_marker`, `is_look_alike` | Anchor and condition markers (§5.7, §15.3) | | `find_markers(document)`, `FoundMarker`, `is_include_only` | Every marker and look-alike in a document's body, placed or not; `FoundMarker.placed(template)` says which apply (`validate` gives the placed ones as `result.index.placed_markers`) | | `Fragment`, `ListFragment`, `block_fragments`, `list_fragments`, `text_fragments`, `list_items`, `item_text` | Where a block's text is, and a list's items | +| `collect_definitions(document)` → `DefinitionRef`, `definition_lookup`, `id_term` | The definitions a document declares in its own source (§7), in document order, each with where it is; `definition_lookup(refs)` is their id-to-term map as `validate` builds `result.index.definition_lookup` (but for definitions imported from other files); `id_term(id)` is the term an empty one reads as | +| `find_definition_anchors(text)` → `DefinitionAnchor` | Every `{{def:}}` in a text, with the quoted term it anchors and where that term starts (§7.2) | +| `render_block(block)`, `render_item(item)` | The other way: a block written as LegalDown source, and a list item's content as written after its marker, as `serialize` writes them | | `Quote`, `block_quotes`, `is_drafting_note(block)` | The block quotes in a block and whether each is a drafting note; whether a quote block is one (§15.6) | | `quote_blocks(block, depth=0)`, `drafting_note_blocks(block, depth=0)` | The blocks a block quote holds, as the validator reads them, as copies you may change; a drafting note's without its `[!DRAFTING]` marker (§15.6). `depth` is how many list items and quotes the quote is in, the count before entering it (a quote in a list item is at 1); past `MAX_QUOTE_DEPTH` the validator reads a quote as one text, so you get one paragraph of its text as written | | `code_content(block)` → `CodeContent(info, text, fenced)` | A code block read as CommonMark reads it: the info string, and the code without its fences or indentation | @@ -515,14 +529,23 @@ for number, section in enumerate(document.layout().sections): print(document.sections[number].title, section.heading.start, [b.start for b in section.blocks]) ``` -If you import one of these from a private module, use its public home: +If you import one of these from a private module, use its public home. The names that left +`legaldown.__all__` in 0.4.0 are listed too, as guidance only: importing them from `legaldown` keeps +working, without a warning. | From | Use | |---|---| +| `legaldown`: `lex`, `Lexed`, `Directive`, `iter_directives`, `is_escaped`, `collect_source_directives` | `legaldown.syntax` | +| `legaldown`: `Fragment`, `ListFragment`, `block_fragments`, `list_fragments`, `list_items`, `item_text`, `is_drafting_note` | `legaldown.syntax` | +| `legaldown`: `collect_definitions`, `definition_lookup`, `id_term`, `DefinitionRef`, `find_definition_anchors`, `DefinitionAnchor` | `legaldown.syntax` | +| `legaldown`: `render_block`, `render_item` | `legaldown.syntax` | +| `legaldown`: `DIRECTIVE_PARAMS`, `KNOWN_DIRECTIVES`, `DELIMITER_PAIRS`, `slugify_identifier` | `legaldown.grammar` | +| `legaldown`: `document_from_dict(data)`, `document_to_dict(document)` | `Document.from_dict(data)`, `document.to_dict()` | +| `legaldown`: `empty_document`, `BLOCK_DEFAULTS`, `metadata_from_dict`, `section_from_dict`, `block_from_dict`, `side_from_dict`, `party_from_dict` | No other home: they stay importable from `legaldown`; `Document()` is a blank document, and `Document.from_dict` builds every part | | `legaldown.directives`: `PLACEHOLDER_TYPE_PARAMS` | `legaldown.grammar` | | `legaldown.directives`: `format_value` | `legaldown.syntax` | | `legaldown.definitions`: `text_fragments` | `legaldown.syntax` | -| `legaldown.definitions`: `DELIMITER_PAIRS` | `legaldown.grammar` (also top-level) | +| `legaldown.definitions`: `DELIMITER_PAIRS` | `legaldown.grammar` | | `legaldown.markdown`: `FENCE_OPEN_RE`, `HTML_COMMENT_RE`, `LINE_ENDING_RE`, `closes_fence`, `dedent`, `fence_end`, `indent_width`, `strip_text` | `legaldown.syntax` | | `legaldown.markers`: `MARKER_RE`, `Marker`, `format_marker`, `is_look_alike`, `parse_marker` | `legaldown.syntax` | | `legaldown.models`: `LIST_KINDS` | `legaldown.grammar` | @@ -586,7 +609,8 @@ for diagnostic in result.diagnostics: A template is read once; a form is a snapshot of the interview over it, so a front end asks for a new form after every answer — a decision opens or closes the questions under it. `parse_template(text)` -is the same for source text. `load_template` raises `FrontmatterError` for frontmatter that cannot +(or `Template(text)`, which takes the same `resolve=`) is the same for source text; `template.path` is +the file `load_template` read, and `None` for a template from its text. `load_template` raises `FrontmatterError` for frontmatter that cannot be read, as `parse` does. **What a form says.** `questions` are those the assembly reaches given the answers so far: a diff --git a/src/legaldown/__init__.py b/src/legaldown/__init__.py index 4675c9f..c2c349c 100644 --- a/src/legaldown/__init__.py +++ b/src/legaldown/__init__.py @@ -18,40 +18,18 @@ from __future__ import annotations # Assembly (§15.7): a named capability (§17.6) -from .assembly import ( - AssemblyError, - AssemblyResult, - Question, - assemble, - needed_questions, - template_questions, -) - -# Definitions (§7) -from .definitions import ( - DELIMITER_PAIRS, - DefinitionAnchor, - DefinitionRef, - Fragment, - ListFragment, - block_fragments, - collect_definitions, - definition_lookup, - find_definition_anchors, - id_term, - list_fragments, -) -from .directives import ( - DIRECTIVE_PARAMS, - KNOWN_DIRECTIVES, - Directive, - Lexed, - is_escaped, - iter_directives, - lex, -) +from .assembly import AssemblyError, AssemblyResult, Question, assemble, needed_questions, template_questions from .files import LoadFile, file_loader -from .models import ( + +# Compatibility: the ``grammar`` and ``syntax`` names imported here, and the +# dict factories with ``BLOCK_DEFAULTS``, left ``__all__`` in 0.4.0 but still +# import from ``legaldown``, as the same objects and without a warning. Their +# supported home is the tooling API, ``legaldown.grammar`` and +# ``legaldown.syntax``; for ``document_from_dict`` and ``document_to_dict`` it is +# ``Document.from_dict`` and ``Document.to_dict`` (``models`` is internal, and +# the other factories and ``empty_document`` have no home but this one). +from .grammar import DELIMITER_PAIRS, DIRECTIVE_PARAMS, KNOWN_DIRECTIVES, slugify_identifier # noqa: F401 +from .models import ( # noqa: F401 -- BLOCK_DEFAULTS and the *_from_dict factories are kept for compatibility BLOCK_DEFAULTS, Amends, Attachment, @@ -68,8 +46,6 @@ document_from_dict, document_to_dict, empty_document, - item_text, - list_items, metadata_from_dict, party_from_dict, section_from_dict, @@ -77,16 +53,40 @@ ) # Parser & serializer -from .parser import FrontmatterError, collect_source_directives, load, parse, parse_document -from .serializer import render_block, render_item, save, serialize, serialize_document +from .parser import FrontmatterError, load, parse, parse_document +from .serializer import save, serialize, serialize_document # The LegalDown specification version this implementation targets. from .specification import SPEC_VERSION +from .syntax import ( # noqa: F401 -- kept for compatibility + DefinitionAnchor, + DefinitionRef, + Directive, + Fragment, + Lexed, + ListFragment, + block_fragments, + collect_definitions, + collect_source_directives, + definition_lookup, + find_definition_anchors, + id_term, + is_drafting_note, + is_escaped, + item_text, + iter_directives, + lex, + list_fragments, + list_items, + render_block, + render_item, +) from .template import AnswersError, Form, Template, load_answers, load_template, parse_template # Validator from .validator import ( AttachmentDefinitionsImporter, + Blank, DefinitionsImporter, Diagnostic, DocumentIndex, @@ -94,9 +94,7 @@ PlacedMarker, SectionIndexEntry, ValidationResult, - is_drafting_note, is_template, - slugify_identifier, validate, validate_document, ) @@ -145,65 +143,29 @@ "Question", "LoadFile", "file_loader", - # Definitions (§7) - "collect_definitions", - "definition_lookup", - "id_term", - "DefinitionRef", - "DELIMITER_PAIRS", - "find_definition_anchors", - "DefinitionAnchor", - # Directives (§11): the lexer, and where text holding them is - "iter_directives", - "lex", - "Lexed", - "is_escaped", - "block_fragments", - "list_fragments", - "Fragment", - "ListFragment", - "Directive", - "DIRECTIVE_PARAMS", - "KNOWN_DIRECTIVES", - "render_block", - "render_item", - "AttachmentDefinitionsImporter", # deprecated since 0.4.0, removed in 0.5.0 # Result types "ValidationResult", "DocumentIndex", "InlineValues", "SectionIndexEntry", + "Blank", "Diagnostic", "PlacedMarker", - # What validate decides, on its own (§15.1, §15.6) + # What validate decides, on its own (§15.1) "is_template", - "is_drafting_note", # Document model "Document", "Metadata", "Section", "Block", "ListItem", - "item_text", - "list_items", "Side", "Party", "Representative", "CustomField", "Amends", "Attachment", - "BLOCK_DEFAULTS", - # Factory functions - "document_from_dict", - "document_to_dict", - "metadata_from_dict", - "section_from_dict", - "block_from_dict", - "side_from_dict", - "party_from_dict", - "empty_document", - # Utilities - "collect_source_directives", - "slugify_identifier", + # Deprecated importer callbacks "DefinitionsImporter", # deprecated since 0.4.0, removed in 0.5.0: use ``validate(resolve=)`` + "AttachmentDefinitionsImporter", # deprecated since 0.4.0, removed in 0.5.0: use ``validate(resolve=)`` ] diff --git a/src/legaldown/models.py b/src/legaldown/models.py index 35a59a0..23de201 100644 --- a/src/legaldown/models.py +++ b/src/legaldown/models.py @@ -1,7 +1,8 @@ """LegalDown document model. Defines the canonical dataclasses for representing a LegalDown document -in memory, plus factory functions for safe construction from dicts. +in memory, plus factory functions for safe construction from dicts (public +as ``Document.from_dict`` and ``Document.to_dict``). """ from __future__ import annotations @@ -157,7 +158,7 @@ class Block: def list_items(block: Block) -> list[ListItem]: """A list's items, each a ``ListItem`` — one written as a string, as a - model built in code may hold it, read as its content (``block_from_dict``).""" + model built in code may hold it, read as its content, as ``Document.from_dict`` reads one.""" return [item if isinstance(item, ListItem) else _list_item(item) for item in block.items] @@ -197,14 +198,37 @@ class Document: #: diagnostics that name their line (§16.9). Only ``parse`` and ``load`` #: set it; it describes the source as parsed, so a document changed afterwards #: may no longer fit it (then its diagnostics name no line). Not part of - #: equality or of ``document_to_dict``. + #: equality or of ``to_dict``. source_map: Any = field(default=None, compare=False, repr=False) #: The file the document was loaded from, absolute (``load`` sets it), for #: resolving the files it refers to. ``None`` for a document #: parsed from a string or built in code. Not part of equality or of - #: ``document_to_dict``: where a document lives is not what it says. + #: ``to_dict``: where a document lives is not what it says. path: Path | None = field(default=None, compare=False) + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> Document: + """A document built from the nested dict *data*, as ``to_dict`` gives + it or as JSON holds it: missing fields take their defaults, values are + read as strings, and a list item may be written as a string, its + content as written after its marker (the item form of models before + 0.3). ``None`` reads as ``{}``. + + A document built from a dict has no ``source_map`` and no ``path``, + and the fields that describe a parsed source, + ``Metadata.not_line_editable`` and ``Metadata.frontmatter_absent``, + are left at their defaults: only ``parse`` and ``load`` set them.""" + return document_from_dict(data) + + def to_dict(self) -> dict[str, Any]: + """The document as a plain, JSON-friendly dict — its metadata, sections, + filename and preamble — that ``Document.from_dict`` reads back. Not + part of it: ``source_map`` and ``path``, which describe where the + document was read from rather than what it says (and + ``Metadata.not_line_editable`` and ``Metadata.frontmatter_absent``, + which it holds but ``from_dict`` does not read).""" + return document_to_dict(self) + def iter_indexed_blocks(self) -> Iterator[tuple[int | None, int, Block]]: """Every body block in document order, as ``(section_index, index, block)``: the preamble first, with ``section_index`` ``None``.""" @@ -599,7 +623,9 @@ def document_to_dict(document: Document) -> dict[str, Any]: def empty_document() -> Document: - """Create a new document with sensible starter content.""" + """A new document with starter content to edit: a provider and a client + side, a definitions section and a scope section. Not a blank document: + ``Document()`` is one, with nothing in it but default metadata.""" return Document( metadata=Metadata( title="Untitled Agreement", diff --git a/src/legaldown/serializer.py b/src/legaldown/serializer.py index 8a140f2..43d2929 100644 --- a/src/legaldown/serializer.py +++ b/src/legaldown/serializer.py @@ -650,7 +650,7 @@ def serialize_document(document: Document) -> str: def render_item(item: ListItem) -> str: """A list item's content as written after its marker, later lines - indented from its content column: the text ``block_from_dict`` reads + indented from its content column: the text ``Document.from_dict`` reads back as the item (a string item).""" return _item_content(item, open_end=True) diff --git a/src/legaldown/syntax.py b/src/legaldown/syntax.py index 275a12c..c987709 100644 --- a/src/legaldown/syntax.py +++ b/src/legaldown/syntax.py @@ -2,9 +2,10 @@ What a tool needs to read the text of a document the way the validator does: the directive lexer (§11.4), anchor and condition markers (§5.7, §15.3), the -text fragments of a block that may hold either, block quotes and drafting -notes (§15.6), and the Markdown helpers (fences, indentation, comments) -those readings are built on. +text fragments of a block that may hold either, the definitions written in +the source (§7), block quotes and drafting notes (§15.6), and the Markdown +helpers (fences, indentation, comments) those readings are built on; and, +the other way, a block or list item written back as LegalDown source. This module is part of the supported API and covered by semantic versioning. Every name here is the validator's own object, not a copy, so a tool that @@ -19,7 +20,19 @@ """ from __future__ import annotations -from .definitions import Fragment, ListFragment, block_fragments, list_fragments, text_fragments +from .definitions import ( + DefinitionAnchor, + DefinitionRef, + Fragment, + ListFragment, + block_fragments, + collect_definitions, + definition_lookup, + find_definition_anchors, + id_term, + list_fragments, + text_fragments, +) from .directives import Directive, Lexed, format_value, is_escaped, iter_directives, lex from .markdown import ( FENCE_OPEN_RE, @@ -43,6 +56,7 @@ quote_blocks, ) from .positions import BlockSpan, HeadingSpan, ItemSpan, SectionSpan, SourceLayout, body_layout +from .serializer import render_block, render_item from .validator.templates import Quote, block_quotes, drafting_note_blocks, is_drafting_note from .validator.units import FoundMarker, find_markers, is_include_only @@ -81,6 +95,16 @@ "CodeContent", "list_items", "item_text", + # Definitions in the source (§7): the terms a document defines, and where + "collect_definitions", + "DefinitionRef", + "definition_lookup", + "id_term", + "find_definition_anchors", + "DefinitionAnchor", + # A block or list item written as LegalDown source + "render_block", + "render_item", # Markdown "FRONTMATTER_RE", "LINE_ENDING_RE", diff --git a/src/legaldown/template.py b/src/legaldown/template.py index f509d2e..8817ae3 100644 --- a/src/legaldown/template.py +++ b/src/legaldown/template.py @@ -208,19 +208,23 @@ class Template: as often as needed. A template is a snapshot: files that change on disk afterwards are not seen — load it again. - *check* is whether ``problems`` include the validator's template rules - (``TEMPLATE_RULES``); ``load_template`` and ``parse_template`` always do. + ``Template(text, resolve=None)`` is ``parse_template``: *resolve* reads the + include fragments and LegalDown attachment files the template names. + ``path`` is the file ``load_template`` read it from, absolute, and ``None`` + for a template made from its text. Raises ``FrontmatterError`` when the template's frontmatter cannot be read. """ - def __init__( - self, text: str, *, resolve: LoadFile | None = None, path: Path | None = None, check: bool = True - ) -> None: - self.path = path + def __init__(self, text: str, *, resolve: LoadFile | None = None, _check: bool = True) -> None: + # Private: ``_check=False`` leaves the validator's template rules + # (``TEMPLATE_RULES``) out of ``problems``, as the deprecated ``assemble`` + # does, for the tests and the conformance harness that check assembly + # on its own. + self.path: Path | None = None self._resolve = _snapshot(resolve) self._t = _read(text, self._resolve) - self._check = check + self._check = _check @cached_property def questions(self) -> tuple[Question, ...]: @@ -460,4 +464,6 @@ def load_template(path: str | os.PathLike[str]) -> Template: read as written — byte-order mark and line endings included — since assembly keeps every byte it does not edit (§15.7.2).""" file = Path(path).absolute() - return Template(file.read_bytes().decode("utf-8"), resolve=file_loader(file.parent), path=file) + template = Template(file.read_bytes().decode("utf-8"), resolve=file_loader(file.parent)) + template.path = file + return template diff --git a/tests/conformance/test_legaldown_fixtures.py b/tests/conformance/test_legaldown_fixtures.py index d3b900f..79ce1bf 100644 --- a/tests/conformance/test_legaldown_fixtures.py +++ b/tests/conformance/test_legaldown_fixtures.py @@ -163,7 +163,7 @@ def _assemble_case_with_a_form(template: Path, answers: Path): """The same by way of ``Template`` and ``Form``, without the validator's template rules: the form and its result.""" loaded = yaml.safe_load(_read(answers)) or {} - form = Template(_read(template), resolve=_loader(template.parent), check=False).form(loaded) + form = Template(_read(template), resolve=_loader(template.parent), _check=False).form(loaded) return form, form.assemble() diff --git a/tests/test_assembly.py b/tests/test_assembly.py index fc228fb..6c9a19b 100644 --- a/tests/test_assembly.py +++ b/tests/test_assembly.py @@ -48,7 +48,7 @@ def _form_assemble(template, answers, *, load_file=None): it must say what the function says: whether assembly can run, and what it writes.""" legacy = legaldown.assemble(template, answers, load_file=load_file) try: - tpl = Template(template, resolve=load_file, check=False) + tpl = Template(template, resolve=load_file, _check=False) except FrontmatterError as exc: return AssemblyResult(diagnostics=[frontmatter_diagnostic(exc)]) form = tpl.form(answers) diff --git a/tests/test_public_api.py b/tests/test_public_api.py index 20e3245..3da3443 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -4,28 +4,34 @@ from __future__ import annotations import dataclasses +import importlib +import inspect +import json import os +import subprocess +import sys +import warnings from pathlib import Path import pytest import legaldown +import legaldown.grammar +import legaldown.models +import legaldown.syntax import legaldown.validator from legaldown import ( + Document, PlacedMarker, + Template, ValidationResult, - block_fragments, - document_from_dict, file_loader, - is_drafting_note, is_template, - list_fragments, load, parse, validate, ) -from legaldown.directives import lex -from legaldown.validator.units import find_markers +from legaldown.syntax import block_fragments, find_markers, is_drafting_note, lex, list_fragments _FRONTMATTER = "---\ntitle: T\n---\n\n" @@ -39,10 +45,138 @@ def test_every_public_name_is_importable(module): assert [name for name in module.__all__ if not hasattr(module, name)] == [] +#: ``legaldown.__all__``: the workflow and the model (0.4.0). +PUBLIC = sorted([ + # Package metadata + "__version__", "SPEC_VERSION", "CONFORMANCE_LEVEL", "CAPABILITIES", + # Core workflow + "load", "parse", "FrontmatterError", "save", "serialize", "validate", + "parse_document", "serialize_document", "validate_document", # deprecated + # Assembly + "load_template", "parse_template", "load_answers", "AnswersError", "Template", "Form", + "Question", "AssemblyResult", "AssemblyError", "LoadFile", "file_loader", + "assemble", "template_questions", "needed_questions", # deprecated + # Result types + "ValidationResult", "DocumentIndex", "InlineValues", "SectionIndexEntry", "Blank", "Diagnostic", + "PlacedMarker", + "is_template", + # Document model + "Document", "Metadata", "Section", "Block", "ListItem", "Side", "Party", "Representative", + "CustomField", "Amends", "Attachment", + "DefinitionsImporter", "AttachmentDefinitionsImporter", # deprecated +]) + +#: What PactTrack and legaldown-render import from ``legaldown`` itself. +_PACTTRACK = [ + "Amends", "AssemblyError", "AssemblyResult", "Attachment", "BLOCK_DEFAULTS", "Block", "CustomField", + "DIRECTIVE_PARAMS", "DefinitionAnchor", "DefinitionRef", "Diagnostic", "Directive", "Document", + "KNOWN_DIRECTIVES", "Lexed", "ListItem", "LoadFile", "Metadata", "Party", "Question", "Representative", + "SPEC_VERSION", "Section", "SectionIndexEntry", "Side", "ValidationResult", "assemble", "block_fragments", + "block_from_dict", "collect_definitions", "collect_source_directives", "document_from_dict", + "document_to_dict", "empty_document", "find_definition_anchors", "is_drafting_note", "is_escaped", + "is_template", "iter_directives", "lex", "list_items", "metadata_from_dict", "needed_questions", + "parse_document", "party_from_dict", "render_block", "section_from_dict", "serialize_document", + "side_from_dict", "slugify_identifier", "template_questions", "validate_document", +] +_RENDER = [ + "AssemblyError", "Block", "Diagnostic", "Directive", "Document", "PlacedMarker", "ValidationResult", + "assemble", "find_definition_anchors", "is_drafting_note", "lex", "list_items", "parse_document", + "render_block", "slugify_identifier", "validate_document", +] +DOWNSTREAM_TOP_LEVEL = sorted(set(_PACTTRACK) | set(_RENDER)) + +#: The names that left ``legaldown.__all__`` in 0.4.0 and still import from +#: ``legaldown``, by their supported home. +MOVED = { + **dict.fromkeys([ + "lex", "Lexed", "Directive", "iter_directives", "is_escaped", "collect_source_directives", + "Fragment", "ListFragment", "block_fragments", "list_fragments", "list_items", "item_text", + "is_drafting_note", "collect_definitions", "definition_lookup", "id_term", "DefinitionRef", + "find_definition_anchors", "DefinitionAnchor", "render_block", "render_item", + ], "legaldown.syntax"), + **dict.fromkeys(["DIRECTIVE_PARAMS", "KNOWN_DIRECTIVES", "DELIMITER_PAIRS", "slugify_identifier"], + "legaldown.grammar"), + # ``Document.from_dict`` and ``Document.to_dict`` stand for the first two; the + # others have no public home but ``legaldown`` (``models`` is internal). + **dict.fromkeys([ + "document_from_dict", "document_to_dict", "metadata_from_dict", "section_from_dict", + "block_from_dict", "side_from_dict", "party_from_dict", "empty_document", "BLOCK_DEFAULTS", + ], "legaldown.models"), +} + + +def test_the_public_names_are_the_workflow_and_the_model(): + assert sorted(legaldown.__all__) == PUBLIC + assert len(set(legaldown.__all__)) == len(legaldown.__all__) + + +def test_a_star_import_is_exactly_the_public_names(): + namespace: dict = {} + exec("from legaldown import *", namespace) + assert sorted(name for name in namespace if name != "__builtins__") == PUBLIC + + +def test_blank_is_the_validators(): + assert legaldown.Blank is legaldown.validator.Blank + + +def test_every_name_that_moved_out_is_its_homes_object(): + assert set(MOVED).isdisjoint(legaldown.__all__) + for name, home in MOVED.items(): + module = importlib.import_module(home) + assert getattr(legaldown, name) is getattr(module, name), name + if home != "legaldown.models": + assert name in module.__all__, name + + +@pytest.mark.parametrize("name", DOWNSTREAM_TOP_LEVEL) +def test_a_downstream_top_level_import_keeps_working_without_a_warning(name): + """Importing never warns; the deprecated callables warn only when called.""" + with warnings.catch_warnings(): + warnings.simplefilter("error") + namespace: dict = {} + exec(f"from legaldown import {name}", namespace) + assert namespace[name] is getattr(legaldown, name) + assert name in legaldown.__all__ or name in MOVED + + +def test_downstream_top_level_imports_in_a_fresh_interpreter_raise_no_warning(): + """``import legaldown`` itself, and every name downstream imports, with + each DeprecationWarning an error.""" + code = f"from legaldown import {', '.join(DOWNSTREAM_TOP_LEVEL)}" + env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parent.parent / "src")} + done = subprocess.run([sys.executable, "-W", "error", "-c", code], capture_output=True, env=env, check=False) + assert done.returncode == 0, done.stderr.decode() + + +def test_the_dict_factories_are_the_documents_methods(): + document = parse(_FRONTMATTER + "Pre.\n\n# A {#a}\n\n- x\n - y\n\n| a |\n|---|\n| b |\n") + data = document.to_dict() + assert data == legaldown.models.document_to_dict(document) + assert json.loads(json.dumps(data)) == data + rebuilt = Document.from_dict(json.loads(json.dumps(data))) + assert rebuilt == legaldown.models.document_from_dict(data) == document + assert (rebuilt.source_map, rebuilt.path) == (None, None) + assert rebuilt.to_dict() == data + assert Document.from_dict(None) == legaldown.models.document_from_dict(None) + + +def test_a_blank_document_is_document_not_the_starter_one(): + assert validate(Document()).is_valid and Document().sections == [] + assert legaldown.empty_document().sections != [] + + +def test_a_template_takes_its_text_and_a_resolver(): + public = [name for name in inspect.signature(Template).parameters if not name.startswith("_")] + assert public == ["text", "resolve"] + + def test_the_renderers_names_are_public(): - for name in ("PlacedMarker", "is_template", "is_drafting_note", "lex", "Lexed", "is_escaped", - "block_fragments", "list_fragments", "list_items"): + for name in ("PlacedMarker", "is_template"): assert name in legaldown.__all__ + for name in ("is_drafting_note", "lex", "Lexed", "is_escaped", "block_fragments", "list_fragments", + "list_items", "find_definition_anchors", "render_block"): + assert name in legaldown.syntax.__all__ for name in ("parse_condition", "Condition", "condition_problem", "exclusive", "Presence", "ALWAYS", "is_template", "PlacedMarker", "IDENTIFIER_RE", "KNOWN_CURRENCIES", "is_valid_iso_date", "is_valid_money_amount", "is_positive_numeric"): @@ -101,7 +235,7 @@ def test_a_marker_out_of_place_is_not_placed(body): def test_a_document_built_in_code_has_no_lines(): - document = document_from_dict({"sections": [{"title": "A", "blocks": [{"kind": "paragraph", "text": "Text {#t}"}]}]}) + document = Document.from_dict({"sections": [{"title": "A", "blocks": [{"kind": "paragraph", "text": "Text {#t}"}]}]}) [marker] = validate(document).index.placed_markers assert (marker.identifier, marker.line) == ("t", None) @@ -184,7 +318,7 @@ def test_a_document_changed_since_parsing_has_no_lines(): def test_a_list_of_string_items_built_in_code(): - document = document_from_dict({"sections": [{"title": "A", "blocks": [ + document = Document.from_dict({"sections": [{"title": "A", "blocks": [ {"kind": "unordered_list", "items": ["a {#x}", "b\n\n - c {#y}"]}, ]}]}) assert [(m.identifier, m.item) for m in validate(document).index.placed_markers] == [("x", 0), ("y", 2)] diff --git a/tests/test_tooling_surface.py b/tests/test_tooling_surface.py index 51b3afe..36ff14e 100644 --- a/tests/test_tooling_surface.py +++ b/tests/test_tooling_surface.py @@ -10,7 +10,7 @@ import legaldown import legaldown.grammar as grammar import legaldown.syntax as syntax -from legaldown import load_answers, parse, validate +from legaldown import load_answers, parse, serialize, validate from legaldown.directives import lex from legaldown.syntax import find_markers from legaldown.validator.helpers import generate_identifier @@ -40,6 +40,9 @@ "Fragment", "ListFragment", "block_fragments", "list_fragments", "text_fragments", "Quote", "block_quotes", "is_drafting_note", "quote_blocks", "drafting_note_blocks", "code_content", "CodeContent", "list_items", "item_text", + "collect_definitions", "DefinitionRef", "definition_lookup", "id_term", + "find_definition_anchors", "DefinitionAnchor", + "render_block", "render_item", "FRONTMATTER_RE", "LINE_ENDING_RE", "HTML_COMMENT_RE", "FENCE_OPEN_RE", "closes_fence", "fence_end", "dedent", "indent_width", "strip_text", "body_layout", "SourceLayout", "SectionSpan", "HeadingSpan", "BlockSpan", "ItemSpan", @@ -48,6 +51,14 @@ #: Where each public name is implemented. IMPLEMENTATION = { "DELIMITER_PAIRS": "legaldown.definitions", + "DefinitionAnchor": "legaldown.definitions", + "DefinitionRef": "legaldown.definitions", + "collect_definitions": "legaldown.definitions", + "definition_lookup": "legaldown.definitions", + "find_definition_anchors": "legaldown.definitions", + "id_term": "legaldown.definitions", + "render_block": "legaldown.serializer", + "render_item": "legaldown.serializer", "Fragment": "legaldown.definitions", "ListFragment": "legaldown.definitions", "block_fragments": "legaldown.definitions", @@ -192,6 +203,30 @@ def test_all_lists_are_the_expected_names(): def test_the_tooling_names_stay_out_of_the_top_level_package(): top = set(legaldown.__all__) assert top.isdisjoint({"PARTY_TYPES", "MAX_SECTION_LEVEL", "choose_problem", "find_markers"}) + # Since 0.4.0 none is listed there: those that were still import from it (``test_public_api``). + assert top.isdisjoint(set(grammar.__all__) - {"SPEC_VERSION"}) + assert top.isdisjoint(syntax.__all__) + + +def test_the_definitions_read_from_source_are_the_validators(): + document = parse('---\ntitle: T\n---\n\n# A\n\n"Late Fee" {{def: late-fee}} means x. "" {{def: bad_id}}\n') + refs = syntax.collect_definitions(document) + assert [(ref.id, ref.term, ref.inline) for ref in refs] == [("late-fee", "Late Fee", False), ("bad_id", "", True)] + assert syntax.definition_lookup(refs) == {"late-fee": "Late Fee"} == validate(document).index.definition_lookup + assert syntax.id_term("late-fee") == "Late Fee" + text = 'A "Late Fee" {{def: late-fee}} is due.' + [anchor] = syntax.find_definition_anchors(text) + assert (anchor.term, anchor.directive.positional, anchor.start) == ("Late Fee", "late-fee", text.index('"')) + + +def test_a_block_and_an_item_are_written_as_the_serializer_writes_them(): + document = parse("---\ntitle: T\n---\n\n# A\n\nText **bold**.\n\n- one\n\n two\n") + [paragraph, listing] = document.sections[0].blocks + assert syntax.render_block(paragraph) == "Text **bold**." + item = syntax.render_item(listing.items[0]) + assert item == "one\n\ntwo" # later lines are indented from the item's content column + assert syntax.render_block(listing) == "- one\n\n two" + assert serialize(document).endswith(syntax.render_block(paragraph) + "\n\n" + syntax.render_block(listing) + "\n") @pytest.mark.parametrize("module", [grammar, syntax])