diff --git a/README.md b/README.md index 8e0dcb6..2f92595 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ integrations, and dashboards: ```json { "legaldown_spec": "0.2", - "validator_version": "0.2.0", + "validator_version": "0.3.0", "diagnostics": [ { "file": "contract.lgd", @@ -287,8 +287,8 @@ party's custom fields (§3.4) are the keys of its object that are none of its fi read into `custom_fields` and written back as keys of their own, a row whose label is still empty as the key `''`. When keys cannot hold them all (a label written twice, or naming one of the party's fields), they are written as the party's `custom_fields` list of `label` and -`value` entries, which is read too. A value that is a list or a mapping is not kept: a custom -field holds text. +`value` entries, which is read too; a key named `custom_fields` is that list's, never a custom +field. A value that is a list or a mapping is not kept: a custom field holds text. A list block's `items` are `ListItem`s, each holding the blocks of its content in order: its first paragraph, then later paragraphs, headings, nested lists (a list block of their own kind), diff --git a/src/legaldown/__init__.py b/src/legaldown/__init__.py index 86b4c53..eae6f84 100644 --- a/src/legaldown/__init__.py +++ b/src/legaldown/__init__.py @@ -97,7 +97,7 @@ validate_document, ) -__version__ = "0.2.0" +__version__ = "0.3.0" #: Conformance level per specification §17. "core" — parse and validate a #: single document. Rendering and Full (multi-file: includes, attachments, diff --git a/src/legaldown/parser.py b/src/legaldown/parser.py index f8d8b5a..b653fe6 100644 --- a/src/legaldown/parser.py +++ b/src/legaldown/parser.py @@ -599,7 +599,7 @@ def _scan_list( lazy = paragraph = open_paragraph = table = False # the blank line closed the item's paragraph quote = previous = None # a later paragraph in the item is not the quote's continue - was_open, was_table = open_paragraph, table # before this line + was_open, was_table, was_paragraph = open_paragraph, table, paragraph # before this line marker = None if RULE_RE.match(line) else LIST_ITEM_RE.match(line) quote_lazy = False # the line lazily continues the item's quote indented = indent_width(line) >= 2 @@ -758,7 +758,12 @@ def _scan_list( and SETEXT_UNDERLINE_RE.match(line.lstrip(" \t")) is not None ) prose = lazy and quote is None and not code and not table and not underline - paragraph = prose and (joined or _is_paragraph_text(content)) + # Not the item's own when the line continues the paragraph of an item + # nested on its first line (``- 1. a`` and `` b``): a sibling of + # that nested item then starts a list, as after no text at all. + paragraph = prose and (joined or _is_paragraph_text(content)) and ( + joined or marker is not None or was_paragraph or not was_open + ) # Open in the item or in an item nested on its line: a lazy line # continues either. open_paragraph = prose and (joined or _is_paragraph_text(content, nested=True)) diff --git a/src/legaldown/positions.py b/src/legaldown/positions.py index 934efa8..a8c9621 100644 --- a/src/legaldown/positions.py +++ b/src/legaldown/positions.py @@ -41,6 +41,11 @@ def document_shape(document: Document) -> tuple[Any, ...]: ) +#: The last element of a mapping's path in ``frontmatter_keys`` when it +#: merges keys in (``<<``): at the merge key's line. +MERGE = object() + + def frontmatter_keys(root: yaml.Node, first_line: int) -> dict[tuple[Any, ...], int]: """The line of each node in the frontmatter's YAML *root*, by its path: a mapping key's line, and a sequence entry's (its ``-`` line). A @@ -48,7 +53,8 @@ def frontmatter_keys(root: yaml.Node, first_line: int) -> dict[tuple[Any, ...], (``models``: a side, a party, an attachment, a representative or a custom field that is not a mapping is left out), so a path indexes the model's lists. *first_line* is the file line of the YAML's first line. - A merged or aliased node is where it is written, not where it is used.""" + A merged or aliased node is where it is written, not where it is used; + a mapping that merges keys in has ``(*path, MERGE)`` too.""" keys: dict[tuple[Any, ...], int] = {} def walk(node: yaml.Node, path: tuple[Any, ...], seen: frozenset[int]) -> None: @@ -60,6 +66,7 @@ def walk(node: yaml.Node, path: tuple[Any, ...], seen: frozenset[int]) -> None: for key, value in node.value: if isinstance(key, yaml.ScalarNode) and key.tag == "tag:yaml.org,2002:merge": merged.append(value) # its keys are this mapping's, below its own + keys.setdefault((*path, MERGE), key.start_mark.line + first_line) elif isinstance(key, yaml.ScalarNode): inner = (*path, key.value) keys.setdefault(inner, key.start_mark.line + first_line) @@ -272,6 +279,18 @@ def has(self, *path: Any) -> bool: """True if the frontmatter node at *path* is written.""" return self._map is not None and path in self._map.keys + def merges(self, *path: Any) -> bool: + """True if the frontmatter mapping at *path* merges keys in (``<<``).""" + return self._map is not None and (*path, MERGE) in self._map.keys + + def children(self, *path: Any) -> list[str]: + """The keys written in the frontmatter mapping at *path*, as written + (a padded key keeps its spaces), in the order of the source.""" + if self._map is None: + return [] + n = len(path) + return [k[n] for k in self._map.keys if len(k) == n + 1 and k[:n] == path and isinstance(k[n], str)] + def field(self, name: str) -> int | None: """A top-level frontmatter key's line, or the frontmatter's first key's when it is not written: where it would go.""" diff --git a/src/legaldown/validator/core.py b/src/legaldown/validator/core.py index eaf99cd..b2a89f1 100644 --- a/src/legaldown/validator/core.py +++ b/src/legaldown/validator/core.py @@ -169,13 +169,31 @@ def _frontmatter_fields( # A custom field is a key of the party's own (§3.4), or an entry # of its `custom_fields` list. Its label, a key, holds no # placeholder; its value is a value field. - for k, cf in enumerate(party.custom_fields): - listed = (*party_path, "custom_fields", k) + # A key's label is read stripped (``models.party_from_dict``), the + # key is written as it is: a label's nth field is the nth key + # that strips to it, in the order of the source — when each has + # one, and none may be merged in (``<<``), which the model reads + # before the party's own: else it is looked for by its label. + written: dict[str, list[str]] = {} + for raw in where.children(*party_path) if where is not None else (): + written.setdefault(raw.strip(), []).append(raw) + listed_paths = [(*party_path, "custom_fields", k) for k in range(len(party.custom_fields))] + keyed_fields = [ + cf.label for cf, listed in zip(party.custom_fields, listed_paths, strict=True) + if where is None or not where.has(*listed) + ] + merges = where is not None and where.merges(*party_path) + for label in set(keyed_fields): + count = keyed_fields.count(label) + if len(written.get(label, ())) != count or (merges and count > 1): + written.pop(label, None) + for cf, listed in zip(party.custom_fields, listed_paths, strict=True): keyed = where is None or not where.has(*listed) + key = written[cf.label].pop(0) if keyed and written.get(cf.label) else cf.label structural.append(( - "a party's custom field name", cf.label, (*party_path, cf.label) if keyed else (*listed, "label") + "a party's custom field name", cf.label, (*party_path, key) if keyed else (*listed, "label") )) - values.append((cf.value, (*party_path, cf.label) if keyed else (*listed, "value"))) + values.append((cf.value, (*party_path, key) if keyed else (*listed, "value"))) return structural, values diff --git a/tests/test_spec_alignment.py b/tests/test_spec_alignment.py index bab014a..2f2c64b 100644 --- a/tests/test_spec_alignment.py +++ b/tests/test_spec_alignment.py @@ -3631,3 +3631,122 @@ def test_a_custom_fields_value_is_checked_as_a_partys_other_values_are(): ) [diagnostic] = [d for d in validate_document(parse_document(source)).diagnostics if d.rule == "directive-unknown-param"] assert diagnostic.line == 9 + + +def _custom_field_diagnostic_line(entry: str) -> tuple[int, str]: + """The line of the one directive-unknown-param in a party written with + *entry* (lines of frontmatter), and that line's text.""" + source = ( + "---\ntitle: T\nsides:\n - name: a\n parties:\n - name: x\n type: legal_entity\n" + " legal_name: X\n" + entry + _SIDE_B + "---\n\n# A\n\nText.\n" + ) + [diagnostic] = [d for d in validate_document(parse_document(source)).diagnostics if d.rule == "directive-unknown-param"] + return diagnostic.line, source.split("\n")[diagnostic.line - 1] + + +def test_a_custom_field_key_with_spaces_is_reported_on_its_own_line(): + """The key is read stripped (§3.4) and written as it is: its line is its own.""" + line, text = _custom_field_diagnostic_line(" ' tax ': '{{placeholder: p, bogus=1}}'\n") + assert line == 9 + assert text.strip().startswith("' tax '") + + +def test_padded_and_plain_custom_field_keys_each_keep_their_own_line(): + line, text = _custom_field_diagnostic_line( + " ' tax ': plain\n tax: '{{placeholder: p, bogus=1}}'\n" + ) + assert (line, text.strip().startswith("tax:")) == (10, True) + line, text = _custom_field_diagnostic_line( + " ' tax ': '{{placeholder: p, bogus=1}}'\n tax: plain\n" + ) + assert (line, text.strip().startswith("' tax '")) == (9, True) + + +def test_a_padded_key_naming_a_party_field_is_no_custom_field_and_moves_no_line(): + """``' name'`` is ignored (§3.4): nothing is checked, or placed, by it.""" + source = ( + "---\ntitle: T\nsides:\n - name: a\n parties:\n - ' name': '{{placeholder: p, bogus=1}}'\n" + " type: legal_entity\n" + _SIDE_B + "---\n\n# A\n\nText.\n" + ) + document = parse_document(source) + assert document.metadata.sides[0].parties[0].custom_fields == [] + assert "directive-unknown-param" not in validate_document(document).rules() + + +def test_custom_field_lines_are_unchanged_for_plain_keys_and_the_list_form(): + line, text = _custom_field_diagnostic_line(" tax_id: '{{placeholder: p, bogus=1}}'\n") + assert (line, text.strip().startswith("tax_id:")) == (9, True) + line, text = _custom_field_diagnostic_line( + " custom_fields:\n - label: tax\n value: '{{placeholder: p, bogus=1}}'\n" + ) + assert (line, text.strip().startswith("value:")) == (11, True) + + +def _last_code_text(body): + """The text of the code block a document's list ends in, at any depth.""" + blocks = parse_document("---\ntitle: T\n---\n\n" + body).preamble + while blocks[-1].kind != "code": + blocks = blocks[-1].items[-1].blocks + return blocks[-1].text + + +@pytest.mark.parametrize( + ("shape", "expected"), + [ + # An unclosed fence ends an item that holds it, the blank line after + # it going into the code: whatever the item's earlier siblings hold. + ("- 1. z\n 2. ```\n\n", "```\n"), + ("- 1. z\n a\n 2. ```\n\n", "```\n"), + ("- 1. z\na\n 2. ```\n\n", "```\n"), + ("- 1. z\n a\n 2. y\n b\n 3. ```\n\n", "```\n"), + ("- 1. z\n a\n 2. ```\n\n\n", "```\n\n"), + ("- 1. z\n a\n 2. ```\n code\n\n", "```\ncode\n"), + ("- - a\n b\n - ```\n\n", "```\n"), + ("- - - a\n b\n - ```\n\n", "```\n"), + ("1. z\n a\n2. ```\n\n", "```\n"), + ("- a\n b\n- ```\n\n", "```\n"), + ], +) +def test_a_fence_left_open_in_a_list_item_keeps_its_blank_lines_whatever_the_siblings_hold(shape, expected): + """A continuation line of an earlier sibling nested on its item's first + line (``- 1. z`` and `` a``) is that sibling's paragraph, not the + enclosing item's own: a later ``2. ```` still starts an item, and the + scan follows its fence to the document's end.""" + assert _last_code_text(shape) == expected + + +@pytest.mark.parametrize( + "source", + [ + "* 2. z\na\n 1. ```\n\n", + "- 1. z\n a\n 2. ```\n\n", + "- 1. z\n a\n 2. ```\n\nafter\n", + "- - - a\n b\n - ```\n\n", + ], +) +def test_a_fence_left_open_in_a_nested_item_survives_the_round_trip(source): + document = parse_document("---\ntitle: T\n---\n\n" + source) + again = parse_document(serialize_document(document)) + assert again == document + assert serialize_document(again) == serialize_document(document) + + +def test_an_item_nested_after_the_first_line_text_of_an_item_that_only_nests_ends_a_paragraph_no_more_than_before(): + """The text of a nested item's paragraph does not make ``2.`` text of the + enclosing item's own paragraph, which no ordered item but ``1.`` may + interrupt: after ``- a``, `` 2. b`` is text.""" + assert [b.kind for b in parse_document("---\ntitle: T\n---\n\n- a\n 2. b\n").preamble[0].items[0].blocks] == ["paragraph"] + nested = parse_document("---\ntitle: T\n---\n\n- 1. z\n a\n 2. b\n").preamble[0].items[0].blocks + assert [b.kind for b in nested] == ["ordered_list"] and len(nested[0].items) == 2 + + +def test_a_custom_field_merged_in_keeps_its_line_next_to_a_padded_key(): + """A field merged in (``<<``) has no key of its own: the padded key's + line is not given to it.""" + source = ( + "---\ntitle: T\nsides:\n - name: a\n parties:\n - name: x\n type: legal_entity\n" + " legal_name: X\n <<: {tax: '{{placeholder: p, bogus=1}}'}\n ' tax ': ok\n" + + _SIDE_B + "---\n\n# A\n\nText.\n" + ) + [diagnostic] = [d for d in validate_document(parse_document(source)).diagnostics if d.rule == "directive-unknown-param"] + assert source.split("\n")[diagnostic.line - 1].strip().startswith("<<:") diff --git a/w_mn.log b/w_mn.log deleted file mode 100644 index c0d7355..0000000 --- a/w_mn.log +++ /dev/null @@ -1,5 +0,0 @@ -Traceback (most recent call last): - File "/tmp/claude-0/review10/worker.py", line 33, in - bodies = json.load(open(sys.argv[2])) - ^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'bodies1.json'