From f3aff89565a9490dcc97ed585c3e428316e987b9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 05:39:47 +0000 Subject: [PATCH 1/4] Report a padded custom field key on its own line; document the reserved custom_fields key (#83) A party key with surrounding spaces is read stripped as a custom field, but the source map records it as written, so its diagnostics fell back to the party's first line. The validator now looks the path up by the key as written (Locator.children), the nth field of a label being the nth key that strips to it. What is read into the model is unchanged. README: a party key named custom_fields is the list form's, never a custom field. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012pWb8FGHgaM3ZWbzsPiKsk --- README.md | 3 +- src/legaldown/positions.py | 8 ++++++ src/legaldown/validator/core.py | 11 ++++++-- tests/test_spec_alignment.py | 49 +++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8e0dcb6..408969e 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,8 @@ party's custom fields (§3.4) are the keys of its object that are none of its fi read into `custom_fields` and written back as keys of their own, a row whose label is still empty as the key `''`. When keys cannot hold them all (a label written twice, or naming one of the party's fields), they are written as the party's `custom_fields` list of `label` and -`value` entries, which is read too. A value that is a list or a mapping is not kept: a custom +`value` entries, which is read too; a key named `custom_fields` is that list's, never a custom +field. A value that is a list or a mapping is not kept: a custom field holds text. A list block's `items` are `ListItem`s, each holding the blocks of its content in order: its diff --git a/src/legaldown/positions.py b/src/legaldown/positions.py index 934efa8..02fc83f 100644 --- a/src/legaldown/positions.py +++ b/src/legaldown/positions.py @@ -272,6 +272,14 @@ def has(self, *path: Any) -> bool: """True if the frontmatter node at *path* is written.""" return self._map is not None and path in self._map.keys + def children(self, *path: Any) -> list[str]: + """The keys written in the frontmatter mapping at *path*, as written + (a padded key keeps its spaces), in the order of the source.""" + if self._map is None: + return [] + n = len(path) + return [k[n] for k in self._map.keys if len(k) == n + 1 and k[:n] == path and isinstance(k[n], str)] + def field(self, name: str) -> int | None: """A top-level frontmatter key's line, or the frontmatter's first key's when it is not written: where it would go.""" diff --git a/src/legaldown/validator/core.py b/src/legaldown/validator/core.py index eaf99cd..61a2717 100644 --- a/src/legaldown/validator/core.py +++ b/src/legaldown/validator/core.py @@ -169,13 +169,20 @@ def _frontmatter_fields( # A custom field is a key of the party's own (§3.4), or an entry # of its `custom_fields` list. Its label, a key, holds no # placeholder; its value is a value field. + # A key's label is read stripped (``models.party_from_dict``), the + # key is written as it is: a label's nth field is the nth key + # that strips to it, in the order of the source. + written: dict[str, list[str]] = {} + for raw in where.children(*party_path) if where is not None else (): + written.setdefault(raw.strip(), []).append(raw) for k, cf in enumerate(party.custom_fields): listed = (*party_path, "custom_fields", k) keyed = where is None or not where.has(*listed) + key = written[cf.label].pop(0) if keyed and written.get(cf.label) else cf.label structural.append(( - "a party's custom field name", cf.label, (*party_path, cf.label) if keyed else (*listed, "label") + "a party's custom field name", cf.label, (*party_path, key) if keyed else (*listed, "label") )) - values.append((cf.value, (*party_path, cf.label) if keyed else (*listed, "value"))) + values.append((cf.value, (*party_path, key) if keyed else (*listed, "value"))) return structural, values diff --git a/tests/test_spec_alignment.py b/tests/test_spec_alignment.py index bab014a..9ba6c91 100644 --- a/tests/test_spec_alignment.py +++ b/tests/test_spec_alignment.py @@ -3631,3 +3631,52 @@ def test_a_custom_fields_value_is_checked_as_a_partys_other_values_are(): ) [diagnostic] = [d for d in validate_document(parse_document(source)).diagnostics if d.rule == "directive-unknown-param"] assert diagnostic.line == 9 + + +def _custom_field_diagnostic_line(entry: str) -> tuple[int, str]: + """The line of the one directive-unknown-param in a party written with + *entry* (lines of frontmatter), and that line's text.""" + source = ( + "---\ntitle: T\nsides:\n - name: a\n parties:\n - name: x\n type: legal_entity\n" + " legal_name: X\n" + entry + _SIDE_B + "---\n\n# A\n\nText.\n" + ) + [diagnostic] = [d for d in validate_document(parse_document(source)).diagnostics if d.rule == "directive-unknown-param"] + return diagnostic.line, source.split("\n")[diagnostic.line - 1] + + +def test_a_custom_field_key_with_spaces_is_reported_on_its_own_line(): + """The key is read stripped (§3.4) and written as it is: its line is its own.""" + line, text = _custom_field_diagnostic_line(" ' tax ': '{{placeholder: p, bogus=1}}'\n") + assert line == 9 + assert text.strip().startswith("' tax '") + + +def test_padded_and_plain_custom_field_keys_each_keep_their_own_line(): + line, text = _custom_field_diagnostic_line( + " ' tax ': plain\n tax: '{{placeholder: p, bogus=1}}'\n" + ) + assert (line, text.strip().startswith("tax:")) == (10, True) + line, text = _custom_field_diagnostic_line( + " ' tax ': '{{placeholder: p, bogus=1}}'\n tax: plain\n" + ) + assert (line, text.strip().startswith("' tax '")) == (9, True) + + +def test_a_padded_key_naming_a_party_field_is_no_custom_field_and_moves_no_line(): + """``' name'`` is ignored (§3.4): nothing is checked, or placed, by it.""" + source = ( + "---\ntitle: T\nsides:\n - name: a\n parties:\n - ' name': '{{placeholder: p, bogus=1}}'\n" + " type: legal_entity\n" + _SIDE_B + "---\n\n# A\n\nText.\n" + ) + document = parse_document(source) + assert document.metadata.sides[0].parties[0].custom_fields == [] + assert "directive-unknown-param" not in validate_document(document).rules() + + +def test_custom_field_lines_are_unchanged_for_plain_keys_and_the_list_form(): + line, text = _custom_field_diagnostic_line(" tax_id: '{{placeholder: p, bogus=1}}'\n") + assert (line, text.strip().startswith("tax_id:")) == (9, True) + line, text = _custom_field_diagnostic_line( + " custom_fields:\n - label: tax\n value: '{{placeholder: p, bogus=1}}'\n" + ) + assert (line, text.strip().startswith("value:")) == (11, True) From 1f6f32f20771e2bf017aa0f3e92f0fe6ea4d6610 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 05:44:41 +0000 Subject: [PATCH 2/4] Keep a nested item's fence blank lines whatever an earlier sibling holds A continuation line of an item nested on its parent's first line (`- 1. z` and ` a`) set `paragraph` in `_scan_list`, the flag for the current item's own open paragraph. A following ` 2. ```` was then read as text of that paragraph, since `2.` may not interrupt one: the scan never saw the item or its fence, so the blank lines after the unclosed fence fell out of the enclosing item and the code block lost its trailing "\n" (only when an earlier sibling had a continuation line). The paragraph belongs to the nested item, so the line does not set `paragraph`; the sibling starts an item as it does after no continuation line. Parse then agrees with cmark-gfm on a body where a list wrongly swallowed the setext heading after it, and `* 2. z\na\n 1. ```` round-trips. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012pWb8FGHgaM3ZWbzsPiKsk --- src/legaldown/parser.py | 9 ++++-- tests/test_spec_alignment.py | 58 ++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/legaldown/parser.py b/src/legaldown/parser.py index f8d8b5a..b653fe6 100644 --- a/src/legaldown/parser.py +++ b/src/legaldown/parser.py @@ -599,7 +599,7 @@ def _scan_list( lazy = paragraph = open_paragraph = table = False # the blank line closed the item's paragraph quote = previous = None # a later paragraph in the item is not the quote's continue - was_open, was_table = open_paragraph, table # before this line + was_open, was_table, was_paragraph = open_paragraph, table, paragraph # before this line marker = None if RULE_RE.match(line) else LIST_ITEM_RE.match(line) quote_lazy = False # the line lazily continues the item's quote indented = indent_width(line) >= 2 @@ -758,7 +758,12 @@ def _scan_list( and SETEXT_UNDERLINE_RE.match(line.lstrip(" \t")) is not None ) prose = lazy and quote is None and not code and not table and not underline - paragraph = prose and (joined or _is_paragraph_text(content)) + # Not the item's own when the line continues the paragraph of an item + # nested on its first line (``- 1. a`` and `` b``): a sibling of + # that nested item then starts a list, as after no text at all. + paragraph = prose and (joined or _is_paragraph_text(content)) and ( + joined or marker is not None or was_paragraph or not was_open + ) # Open in the item or in an item nested on its line: a lazy line # continues either. open_paragraph = prose and (joined or _is_paragraph_text(content, nested=True)) diff --git a/tests/test_spec_alignment.py b/tests/test_spec_alignment.py index 9ba6c91..20d4288 100644 --- a/tests/test_spec_alignment.py +++ b/tests/test_spec_alignment.py @@ -3680,3 +3680,61 @@ def test_custom_field_lines_are_unchanged_for_plain_keys_and_the_list_form(): " custom_fields:\n - label: tax\n value: '{{placeholder: p, bogus=1}}'\n" ) assert (line, text.strip().startswith("value:")) == (11, True) + + +def _last_code_text(body): + """The text of the code block a document's list ends in, at any depth.""" + blocks = parse_document("---\ntitle: T\n---\n\n" + body).preamble + while blocks[-1].kind != "code": + blocks = blocks[-1].items[-1].blocks + return blocks[-1].text + + +@pytest.mark.parametrize( + ("shape", "expected"), + [ + # An unclosed fence ends an item that holds it, the blank line after + # it going into the code: whatever the item's earlier siblings hold. + ("- 1. z\n 2. ```\n\n", "```\n"), + ("- 1. z\n a\n 2. ```\n\n", "```\n"), + ("- 1. z\na\n 2. ```\n\n", "```\n"), + ("- 1. z\n a\n 2. y\n b\n 3. ```\n\n", "```\n"), + ("- 1. z\n a\n 2. ```\n\n\n", "```\n\n"), + ("- 1. z\n a\n 2. ```\n code\n\n", "```\ncode\n"), + ("- - a\n b\n - ```\n\n", "```\n"), + ("- - - a\n b\n - ```\n\n", "```\n"), + ("1. z\n a\n2. ```\n\n", "```\n"), + ("- a\n b\n- ```\n\n", "```\n"), + ], +) +def test_a_fence_left_open_in_a_list_item_keeps_its_blank_lines_whatever_the_siblings_hold(shape, expected): + """A continuation line of an earlier sibling nested on its item's first + line (``- 1. z`` and `` a``) is that sibling's paragraph, not the + enclosing item's own: a later ``2. ```` still starts an item, and the + scan follows its fence to the document's end.""" + assert _last_code_text(shape) == expected + + +@pytest.mark.parametrize( + "source", + [ + "* 2. z\na\n 1. ```\n\n", + "- 1. z\n a\n 2. ```\n\n", + "- 1. z\n a\n 2. ```\n\nafter\n", + "- - - a\n b\n - ```\n\n", + ], +) +def test_a_fence_left_open_in_a_nested_item_survives_the_round_trip(source): + document = parse_document("---\ntitle: T\n---\n\n" + source) + again = parse_document(serialize_document(document)) + assert again == document + assert serialize_document(again) == serialize_document(document) + + +def test_an_item_nested_after_the_first_line_text_of_an_item_that_only_nests_ends_a_paragraph_no_more_than_before(): + """The text of a nested item's paragraph does not make ``2.`` text of the + enclosing item's own paragraph, which no ordered item but ``1.`` may + interrupt: after ``- a``, `` 2. b`` is text.""" + assert [b.kind for b in parse_document("---\ntitle: T\n---\n\n- a\n 2. b\n").preamble[0].items[0].blocks] == ["paragraph"] + nested = parse_document("---\ntitle: T\n---\n\n- 1. z\n a\n 2. b\n").preamble[0].items[0].blocks + assert [b.kind for b in nested] == ["ordered_list"] and len(nested[0].items) == 2 From 000f6ad1e77e8d97f41769a97010eeda6eac190d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 05:39:38 +0000 Subject: [PATCH 3/4] Prepare release 0.3.0 Bump __version__ to 0.3.0 (the single source of truth, read by pyproject.toml) and the validator_version in the README's JSON example. Specification-version references (0.2) are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012pWb8FGHgaM3ZWbzsPiKsk --- README.md | 2 +- src/legaldown/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 408969e..51ff0c0 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ integrations, and dashboards: ```json { "legaldown_spec": "0.2", - "validator_version": "0.2.0", + "validator_version": "0.3.0", "diagnostics": [ { "file": "contract.lgd", diff --git a/src/legaldown/__init__.py b/src/legaldown/__init__.py index 86b4c53..eae6f84 100644 --- a/src/legaldown/__init__.py +++ b/src/legaldown/__init__.py @@ -97,7 +97,7 @@ validate_document, ) -__version__ = "0.2.0" +__version__ = "0.3.0" #: Conformance level per specification §17. "core" — parse and validate a #: single document. Rendering and Full (multi-file: includes, attachments, From f4a2242319156623da05fc08174e7c9f60088689 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 10:11:43 +0000 Subject: [PATCH 4/4] Give a merged custom field its own line; drop a stray log file A custom field merged into a party (<<) has no key of its own, and the model reads merged fields first: pairing the nth field with the nth key that strips to its label then gave a merged field a padded key's line. The source map records a mapping's merge (positions.MERGE), and where a party merges keys in, a label several fields share is looked up by the label, as before the padded-key change. README reflowed; w_mn.log, a stray fuzzing log committed in #72, removed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012pWb8FGHgaM3ZWbzsPiKsk --- README.md | 3 +-- src/legaldown/positions.py | 13 ++++++++++++- src/legaldown/validator/core.py | 17 ++++++++++++++--- tests/test_spec_alignment.py | 12 ++++++++++++ w_mn.log | 5 ----- 5 files changed, 39 insertions(+), 11 deletions(-) delete mode 100644 w_mn.log diff --git a/README.md b/README.md index 51ff0c0..2f92595 100644 --- a/README.md +++ b/README.md @@ -288,8 +288,7 @@ read into `custom_fields` and written back as keys of their own, a row whose lab empty as the key `''`. When keys cannot hold them all (a label written twice, or naming one of the party's fields), they are written as the party's `custom_fields` list of `label` and `value` entries, which is read too; a key named `custom_fields` is that list's, never a custom -field. A value that is a list or a mapping is not kept: a custom -field holds text. +field. A value that is a list or a mapping is not kept: a custom field holds text. A list block's `items` are `ListItem`s, each holding the blocks of its content in order: its first paragraph, then later paragraphs, headings, nested lists (a list block of their own kind), diff --git a/src/legaldown/positions.py b/src/legaldown/positions.py index 02fc83f..a8c9621 100644 --- a/src/legaldown/positions.py +++ b/src/legaldown/positions.py @@ -41,6 +41,11 @@ def document_shape(document: Document) -> tuple[Any, ...]: ) +#: The last element of a mapping's path in ``frontmatter_keys`` when it +#: merges keys in (``<<``): at the merge key's line. +MERGE = object() + + def frontmatter_keys(root: yaml.Node, first_line: int) -> dict[tuple[Any, ...], int]: """The line of each node in the frontmatter's YAML *root*, by its path: a mapping key's line, and a sequence entry's (its ``-`` line). A @@ -48,7 +53,8 @@ def frontmatter_keys(root: yaml.Node, first_line: int) -> dict[tuple[Any, ...], (``models``: a side, a party, an attachment, a representative or a custom field that is not a mapping is left out), so a path indexes the model's lists. *first_line* is the file line of the YAML's first line. - A merged or aliased node is where it is written, not where it is used.""" + A merged or aliased node is where it is written, not where it is used; + a mapping that merges keys in has ``(*path, MERGE)`` too.""" keys: dict[tuple[Any, ...], int] = {} def walk(node: yaml.Node, path: tuple[Any, ...], seen: frozenset[int]) -> None: @@ -60,6 +66,7 @@ def walk(node: yaml.Node, path: tuple[Any, ...], seen: frozenset[int]) -> None: for key, value in node.value: if isinstance(key, yaml.ScalarNode) and key.tag == "tag:yaml.org,2002:merge": merged.append(value) # its keys are this mapping's, below its own + keys.setdefault((*path, MERGE), key.start_mark.line + first_line) elif isinstance(key, yaml.ScalarNode): inner = (*path, key.value) keys.setdefault(inner, key.start_mark.line + first_line) @@ -272,6 +279,10 @@ def has(self, *path: Any) -> bool: """True if the frontmatter node at *path* is written.""" return self._map is not None and path in self._map.keys + def merges(self, *path: Any) -> bool: + """True if the frontmatter mapping at *path* merges keys in (``<<``).""" + return self._map is not None and (*path, MERGE) in self._map.keys + def children(self, *path: Any) -> list[str]: """The keys written in the frontmatter mapping at *path*, as written (a padded key keeps its spaces), in the order of the source.""" diff --git a/src/legaldown/validator/core.py b/src/legaldown/validator/core.py index 61a2717..b2a89f1 100644 --- a/src/legaldown/validator/core.py +++ b/src/legaldown/validator/core.py @@ -171,12 +171,23 @@ def _frontmatter_fields( # placeholder; its value is a value field. # A key's label is read stripped (``models.party_from_dict``), the # key is written as it is: a label's nth field is the nth key - # that strips to it, in the order of the source. + # that strips to it, in the order of the source — when each has + # one, and none may be merged in (``<<``), which the model reads + # before the party's own: else it is looked for by its label. written: dict[str, list[str]] = {} for raw in where.children(*party_path) if where is not None else (): written.setdefault(raw.strip(), []).append(raw) - for k, cf in enumerate(party.custom_fields): - listed = (*party_path, "custom_fields", k) + listed_paths = [(*party_path, "custom_fields", k) for k in range(len(party.custom_fields))] + keyed_fields = [ + cf.label for cf, listed in zip(party.custom_fields, listed_paths, strict=True) + if where is None or not where.has(*listed) + ] + merges = where is not None and where.merges(*party_path) + for label in set(keyed_fields): + count = keyed_fields.count(label) + if len(written.get(label, ())) != count or (merges and count > 1): + written.pop(label, None) + for cf, listed in zip(party.custom_fields, listed_paths, strict=True): keyed = where is None or not where.has(*listed) key = written[cf.label].pop(0) if keyed and written.get(cf.label) else cf.label structural.append(( diff --git a/tests/test_spec_alignment.py b/tests/test_spec_alignment.py index 20d4288..2f2c64b 100644 --- a/tests/test_spec_alignment.py +++ b/tests/test_spec_alignment.py @@ -3738,3 +3738,15 @@ def test_an_item_nested_after_the_first_line_text_of_an_item_that_only_nests_end assert [b.kind for b in parse_document("---\ntitle: T\n---\n\n- a\n 2. b\n").preamble[0].items[0].blocks] == ["paragraph"] nested = parse_document("---\ntitle: T\n---\n\n- 1. z\n a\n 2. b\n").preamble[0].items[0].blocks assert [b.kind for b in nested] == ["ordered_list"] and len(nested[0].items) == 2 + + +def test_a_custom_field_merged_in_keeps_its_line_next_to_a_padded_key(): + """A field merged in (``<<``) has no key of its own: the padded key's + line is not given to it.""" + source = ( + "---\ntitle: T\nsides:\n - name: a\n parties:\n - name: x\n type: legal_entity\n" + " legal_name: X\n <<: {tax: '{{placeholder: p, bogus=1}}'}\n ' tax ': ok\n" + + _SIDE_B + "---\n\n# A\n\nText.\n" + ) + [diagnostic] = [d for d in validate_document(parse_document(source)).diagnostics if d.rule == "directive-unknown-param"] + assert source.split("\n")[diagnostic.line - 1].strip().startswith("<<:") diff --git a/w_mn.log b/w_mn.log deleted file mode 100644 index c0d7355..0000000 --- a/w_mn.log +++ /dev/null @@ -1,5 +0,0 @@ -Traceback (most recent call last): - File "/tmp/claude-0/review10/worker.py", line 33, in - bodies = json.load(open(sys.argv[2])) - ^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'bodies1.json'