Pre-release fixes and 0.3.0 (#86, #83 follow-ups)

README.md

@@ -147,7 +147,7 @@ integrations, and dashboards:
 ```json
 {
   "legaldown_spec": "0.2",
-  "validator_version": "0.2.0",
+  "validator_version": "0.3.0",
   "diagnostics": [
     {
       "file": "contract.lgd",
@@ -287,8 +287,8 @@ party's custom fields (§3.4) are the keys of its object that are none of its fi
 read into `custom_fields` and written back as keys of their own, a row whose label is still
 empty as the key `''`. When keys cannot hold them all (a label written twice, or naming one of
 the party's fields), they are written as the party's `custom_fields` list of `label` and
-`value` entries, which is read too. A value that is a list or a mapping is not kept: a custom
-field holds text.
+`value` entries, which is read too; a key named `custom_fields` is that list's, never a custom
+field. A value that is a list or a mapping is not kept: a custom field holds text.
 
 A list block's `items` are `ListItem`s, each holding the blocks of its content in order: its
 first paragraph, then later paragraphs, headings, nested lists (a list block of their own kind),

src/legaldown/__init__.py

@@ -97,7 +97,7 @@
     validate_document,
 )
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 
 #: Conformance level per specification §17. "core" — parse and validate a
 #: single document. Rendering and Full (multi-file: includes, attachments,

src/legaldown/parser.py

@@ -599,7 +599,7 @@ def _scan_list(
             lazy = paragraph = open_paragraph = table = False  # the blank line closed the item's paragraph
             quote = previous = None  # a later paragraph in the item is not the quote's
             continue
-        was_open, was_table = open_paragraph, table  # before this line
+        was_open, was_table, was_paragraph = open_paragraph, table, paragraph  # before this line
         marker = None if RULE_RE.match(line) else LIST_ITEM_RE.match(line)
         quote_lazy = False  # the line lazily continues the item's quote
         indented = indent_width(line) >= 2
@@ -758,7 +758,12 @@ def _scan_list(
             and SETEXT_UNDERLINE_RE.match(line.lstrip(" \t")) is not None
         )
         prose = lazy and quote is None and not code and not table and not underline
-        paragraph = prose and (joined or _is_paragraph_text(content))
+        # Not the item's own when the line continues the paragraph of an item
+        # nested on its first line (``- 1. a`` and ``     b``): a sibling of
+        # that nested item then starts a list, as after no text at all.
+        paragraph = prose and (joined or _is_paragraph_text(content)) and (
+            joined or marker is not None or was_paragraph or not was_open
+        )
         # Open in the item or in an item nested on its line: a lazy line
         # continues either.
         open_paragraph = prose and (joined or _is_paragraph_text(content, nested=True))

src/legaldown/positions.py

@@ -41,14 +41,20 @@ def document_shape(document: Document) -> tuple[Any, ...]:
     )
 
 
+#: The last element of a mapping's path in ``frontmatter_keys`` when it
+#: merges keys in (``<<``): at the merge key's line.
+MERGE = object()
+
+
 def frontmatter_keys(root: yaml.Node, first_line: int) -> dict[tuple[Any, ...], int]:
     """The line of each node in the frontmatter's YAML *root*, by its path:
     a mapping key's line, and a sequence entry's (its ``-`` line). A
     sequence's entries are counted as the model counts them, mappings only
     (``models``: a side, a party, an attachment, a representative or a
     custom field that is not a mapping is left out), so a path indexes the
     model's lists. *first_line* is the file line of the YAML's first line.
-    A merged or aliased node is where it is written, not where it is used."""
+    A merged or aliased node is where it is written, not where it is used;
+    a mapping that merges keys in has ``(*path, MERGE)`` too."""
     keys: dict[tuple[Any, ...], int] = {}
 
     def walk(node: yaml.Node, path: tuple[Any, ...], seen: frozenset[int]) -> None:
@@ -60,6 +66,7 @@ def walk(node: yaml.Node, path: tuple[Any, ...], seen: frozenset[int]) -> None:
             for key, value in node.value:
                 if isinstance(key, yaml.ScalarNode) and key.tag == "tag:yaml.org,2002:merge":
                     merged.append(value)  # its keys are this mapping's, below its own
+                    keys.setdefault((*path, MERGE), key.start_mark.line + first_line)
                 elif isinstance(key, yaml.ScalarNode):
                     inner = (*path, key.value)
                     keys.setdefault(inner, key.start_mark.line + first_line)
@@ -272,6 +279,18 @@ def has(self, *path: Any) -> bool:
         """True if the frontmatter node at *path* is written."""
         return self._map is not None and path in self._map.keys
 
+    def merges(self, *path: Any) -> bool:
+        """True if the frontmatter mapping at *path* merges keys in (``<<``)."""
+        return self._map is not None and (*path, MERGE) in self._map.keys
+
+    def children(self, *path: Any) -> list[str]:
+        """The keys written in the frontmatter mapping at *path*, as written
+        (a padded key keeps its spaces), in the order of the source."""
+        if self._map is None:
+            return []
+        n = len(path)
+        return [k[n] for k in self._map.keys if len(k) == n + 1 and k[:n] == path and isinstance(k[n], str)]
+
     def field(self, name: str) -> int | None:
         """A top-level frontmatter key's line, or the frontmatter's first
         key's when it is not written: where it would go."""

src/legaldown/validator/core.py

@@ -169,13 +169,31 @@ def _frontmatter_fields(
             # A custom field is a key of the party's own (§3.4), or an entry
             # of its `custom_fields` list. Its label, a key, holds no
             # placeholder; its value is a value field.
-            for k, cf in enumerate(party.custom_fields):
-                listed = (*party_path, "custom_fields", k)
+            # A key's label is read stripped (``models.party_from_dict``), the
+            # key is written as it is: a label's nth field is the nth key
+            # that strips to it, in the order of the source — when each has
+            # one, and none may be merged in (``<<``), which the model reads
+            # before the party's own: else it is looked for by its label.
+            written: dict[str, list[str]] = {}
+            for raw in where.children(*party_path) if where is not None else ():
+                written.setdefault(raw.strip(), []).append(raw)
+            listed_paths = [(*party_path, "custom_fields", k) for k in range(len(party.custom_fields))]
+            keyed_fields = [
+                cf.label for cf, listed in zip(party.custom_fields, listed_paths, strict=True)
+                if where is None or not where.has(*listed)
+            ]
+            merges = where is not None and where.merges(*party_path)
+            for label in set(keyed_fields):
+                count = keyed_fields.count(label)
+                if len(written.get(label, ())) != count or (merges and count > 1):
+                    written.pop(label, None)
+            for cf, listed in zip(party.custom_fields, listed_paths, strict=True):
                 keyed = where is None or not where.has(*listed)
+                key = written[cf.label].pop(0) if keyed and written.get(cf.label) else cf.label
                 structural.append((
-                    "a party's custom field name", cf.label, (*party_path, cf.label) if keyed else (*listed, "label")
+                    "a party's custom field name", cf.label, (*party_path, key) if keyed else (*listed, "label")
                 ))
-                values.append((cf.value, (*party_path, cf.label) if keyed else (*listed, "value")))
+                values.append((cf.value, (*party_path, key) if keyed else (*listed, "value")))
     return structural, values
 
 

tests/test_spec_alignment.py

@@ -3631,3 +3631,122 @@ def test_a_custom_fields_value_is_checked_as_a_partys_other_values_are():
     )
     [diagnostic] = [d for d in validate_document(parse_document(source)).diagnostics if d.rule == "directive-unknown-param"]
     assert diagnostic.line == 9
+
+
+def _custom_field_diagnostic_line(entry: str) -> tuple[int, str]:
+    """The line of the one directive-unknown-param in a party written with
+    *entry* (lines of frontmatter), and that line's text."""
+    source = (
+        "---\ntitle: T\nsides:\n  - name: a\n    parties:\n      - name: x\n        type: legal_entity\n"
+        "        legal_name: X\n" + entry + _SIDE_B + "---\n\n# A\n\nText.\n"
+    )
+    [diagnostic] = [d for d in validate_document(parse_document(source)).diagnostics if d.rule == "directive-unknown-param"]
+    return diagnostic.line, source.split("\n")[diagnostic.line - 1]
+
+
+def test_a_custom_field_key_with_spaces_is_reported_on_its_own_line():
+    """The key is read stripped (§3.4) and written as it is: its line is its own."""
+    line, text = _custom_field_diagnostic_line("        ' tax ': '{{placeholder: p, bogus=1}}'\n")
+    assert line == 9
+    assert text.strip().startswith("' tax '")
+
+
+def test_padded_and_plain_custom_field_keys_each_keep_their_own_line():
+    line, text = _custom_field_diagnostic_line(
+        "        ' tax ': plain\n        tax: '{{placeholder: p, bogus=1}}'\n"
+    )
+    assert (line, text.strip().startswith("tax:")) == (10, True)
+    line, text = _custom_field_diagnostic_line(
+        "        ' tax ': '{{placeholder: p, bogus=1}}'\n        tax: plain\n"
+    )
+    assert (line, text.strip().startswith("' tax '")) == (9, True)
+
+
+def test_a_padded_key_naming_a_party_field_is_no_custom_field_and_moves_no_line():
+    """``' name'`` is ignored (§3.4): nothing is checked, or placed, by it."""
+    source = (
+        "---\ntitle: T\nsides:\n  - name: a\n    parties:\n      - ' name': '{{placeholder: p, bogus=1}}'\n"
+        "        type: legal_entity\n" + _SIDE_B + "---\n\n# A\n\nText.\n"
+    )
+    document = parse_document(source)
+    assert document.metadata.sides[0].parties[0].custom_fields == []
+    assert "directive-unknown-param" not in validate_document(document).rules()
+
+
+def test_custom_field_lines_are_unchanged_for_plain_keys_and_the_list_form():
+    line, text = _custom_field_diagnostic_line("        tax_id: '{{placeholder: p, bogus=1}}'\n")
+    assert (line, text.strip().startswith("tax_id:")) == (9, True)
+    line, text = _custom_field_diagnostic_line(
+        "        custom_fields:\n          - label: tax\n            value: '{{placeholder: p, bogus=1}}'\n"
+    )
+    assert (line, text.strip().startswith("value:")) == (11, True)
+
+
+def _last_code_text(body):
+    """The text of the code block a document's list ends in, at any depth."""
+    blocks = parse_document("---\ntitle: T\n---\n\n" + body).preamble
+    while blocks[-1].kind != "code":
+        blocks = blocks[-1].items[-1].blocks
+    return blocks[-1].text
+
+
+@pytest.mark.parametrize(
+    ("shape", "expected"),
+    [
+        # An unclosed fence ends an item that holds it, the blank line after
+        # it going into the code: whatever the item's earlier siblings hold.
+        ("- 1. z\n  2. ```\n\n", "```\n"),
+        ("- 1. z\n     a\n  2. ```\n\n", "```\n"),
+        ("- 1. z\na\n  2. ```\n\n", "```\n"),
+        ("- 1. z\n     a\n  2. y\n     b\n  3. ```\n\n", "```\n"),
+        ("- 1. z\n     a\n  2. ```\n\n\n", "```\n\n"),
+        ("- 1. z\n     a\n  2. ```\n     code\n\n", "```\ncode\n"),
+        ("- - a\n    b\n  - ```\n\n", "```\n"),
+        ("- - - a\n      b\n    - ```\n\n", "```\n"),
+        ("1. z\n   a\n2. ```\n\n", "```\n"),
+        ("- a\n  b\n- ```\n\n", "```\n"),
+    ],
+)
+def test_a_fence_left_open_in_a_list_item_keeps_its_blank_lines_whatever_the_siblings_hold(shape, expected):
+    """A continuation line of an earlier sibling nested on its item's first
+    line (``- 1. z`` and ``     a``) is that sibling's paragraph, not the
+    enclosing item's own: a later ``2. ```` still starts an item, and the
+    scan follows its fence to the document's end."""
+    assert _last_code_text(shape) == expected
+
+
+@pytest.mark.parametrize(
+    "source",
+    [
+        "* 2. z\na\n  1. ```\n\n",
+        "- 1. z\n     a\n  2. ```\n\n",
+        "- 1. z\n     a\n  2. ```\n\nafter\n",
+        "- - - a\n      b\n    - ```\n\n",
+    ],
+)
+def test_a_fence_left_open_in_a_nested_item_survives_the_round_trip(source):
+    document = parse_document("---\ntitle: T\n---\n\n" + source)
+    again = parse_document(serialize_document(document))
+    assert again == document
+    assert serialize_document(again) == serialize_document(document)
+
+
+def test_an_item_nested_after_the_first_line_text_of_an_item_that_only_nests_ends_a_paragraph_no_more_than_before():
+    """The text of a nested item's paragraph does not make ``2.`` text of the
+    enclosing item's own paragraph, which no ordered item but ``1.`` may
+    interrupt: after ``- a``, ``  2. b`` is text."""
+    assert [b.kind for b in parse_document("---\ntitle: T\n---\n\n- a\n  2. b\n").preamble[0].items[0].blocks] == ["paragraph"]
+    nested = parse_document("---\ntitle: T\n---\n\n- 1. z\n     a\n  2. b\n").preamble[0].items[0].blocks
+    assert [b.kind for b in nested] == ["ordered_list"] and len(nested[0].items) == 2
+
+
+def test_a_custom_field_merged_in_keeps_its_line_next_to_a_padded_key():
+    """A field merged in (``<<``) has no key of its own: the padded key's
+    line is not given to it."""
+    source = (
+        "---\ntitle: T\nsides:\n  - name: a\n    parties:\n      - name: x\n        type: legal_entity\n"
+        "        legal_name: X\n        <<: {tax: '{{placeholder: p, bogus=1}}'}\n        ' tax ': ok\n"
+        + _SIDE_B + "---\n\n# A\n\nText.\n"
+    )
+    [diagnostic] = [d for d in validate_document(parse_document(source)).diagnostics if d.rule == "directive-unknown-param"]
+    assert source.split("\n")[diagnostic.line - 1].strip().startswith("<<:")