bugfix: metadata parsing

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.1] - 2026-02-10
+
+### Fixed
+
+- **`parse_html` metadata propagation** - Fixed bug where `article`, `group`, and `section` columns were assigned the same values (from the last element in the document) for all rows. The parser now processes `<p>` tags in a single pass in document order, so each row reflects its actual structural position. Preamble/recital rows before any article now correctly have `None` for `article`. ([#1](https://github.com/morrieinmaas/eurlxp/issues/1))
+
 ## [0.4.0] - 2026-02-04
 
 ### Added

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eurlxp"
-version = "0.4.0"
+version = "0.4.1"
 description = "A modern EUR-Lex parser for Python - fetch and parse EU legal documents"
 readme = "README.md"
 license = { text = "MIT" }

src/eurlxp/parser.py

@@ -205,6 +205,9 @@ def _parse_html_with_beautifulsoup(html: str) -> list[ParseResult]:
 
     Tries lxml-xml parser first (for XHTML documents), then falls back to
     lxml HTML parser for older HTML documents.
+
+    Processes all <p> tags in a single pass in document order so that
+    article, group, and section context updates apply only to subsequent rows.
     """
     from bs4 import BeautifulSoup
 
@@ -218,55 +221,47 @@ def _parse_html_with_beautifulsoup(html: str) -> list[ParseResult]:
     results: list[ParseResult] = []
     context = ParseContext()
 
-    # Find document title
-    # Supports: OJ format (doc-ti, oj-doc-ti), Commission format (Titreobjet, Typedudocument)
-    for doc_ti in soup.find_all("p", class_=["doc-ti", "oj-doc-ti", "Titreobjet", "Typedudocument"]):
-        text = doc_ti.get_text(strip=True)
-        if text:
+    doc_title_classes = {"doc-ti", "oj-doc-ti", "Titreobjet", "Typedudocument"}
+    article_title_classes = {"ti-art", "oj-ti-art", "Titrearticle"}
+    text_classes = {"normal", "oj-normal", "Normal"}
+
+    # Single pass through all <p> tags in document order
+    for p_tag in soup.find_all("p"):
+        css_classes = p_tag.get("class") or []
+        if isinstance(css_classes, str):
+            css_classes = [css_classes]
+        css_class_str = " ".join(css_classes)
+        css_class_set = set(css_classes)
+
+        text = p_tag.get_text(strip=True)
+        if not text:
+            continue
+
+        # Document title
+        if css_class_set & doc_title_classes:
             if context.document is None:
                 context.document = ""
             context.document += text
             results.append(ParseResult(text=text, item_type="doc-title", ref=[], context=context.copy()))
 
-    # Find article titles
-    # Supports: OJ format (ti-art, oj-ti-art), Commission format (Titrearticle)
-    for ti_art in soup.find_all("p", class_=["ti-art", "oj-ti-art", "Titrearticle"]):
-        text = ti_art.get_text(strip=True)
-        if text:
+        # Article title
+        elif css_class_set & article_title_classes:
             context.article = text.replace("Article", "").strip()
+            context.paragraph = None
             results.append(ParseResult(text=text, item_type="art-title", ref=[], context=context.copy()))
 
-    # Find group titles (ti-grseq-* classes)
-    for p_tag in soup.find_all("p"):
-        css_class = p_tag.get("class")
-        if css_class is None:
-            css_class = ""
-        elif isinstance(css_class, list):
-            css_class = " ".join(css_class)
-        if css_class and ("ti-grseq-" in css_class or "oj-ti-grseq-" in css_class):
-            text = p_tag.get_text(strip=True)
-            if text:
-                context.group = text
-                results.append(ParseResult(text=text, item_type="group-title", ref=[], context=context.copy()))
+        # Group title (ti-grseq-* or oj-ti-grseq-* classes)
+        elif "ti-grseq-" in css_class_str or "oj-ti-grseq-" in css_class_str:
+            context.group = text
+            results.append(ParseResult(text=text, item_type="group-title", ref=[], context=context.copy()))
 
-    # Find section titles (ti-section-* classes)
-    for p_tag in soup.find_all("p"):
-        css_class = p_tag.get("class")
-        if css_class is None:
-            css_class = ""
-        elif isinstance(css_class, list):
-            css_class = " ".join(css_class)
-        if css_class and ("ti-section-" in css_class or "oj-ti-section-" in css_class):
-            text = p_tag.get_text(strip=True)
-            if text:
-                context.section = text
-                results.append(ParseResult(text=text, item_type="section-title", ref=[], context=context.copy()))
-
-    # Find normal text paragraphs
-    # Supports: OJ format (normal, oj-normal), Commission format (Normal - capital N)
-    for normal in soup.find_all("p", class_=["normal", "oj-normal", "Normal"]):
-        text = normal.get_text(strip=True)
-        if text:
+        # Section title (ti-section-* or oj-ti-section-* classes)
+        elif "ti-section-" in css_class_str or "oj-ti-section-" in css_class_str:
+            context.section = text
+            results.append(ParseResult(text=text, item_type="section-title", ref=[], context=context.copy()))
+
+        # Normal text paragraphs
+        elif css_class_set & text_classes:
             # Check for numbered paragraphs
             match = re.match(r"^[(]?([0-9]+)[).]?", text)
             if match:

tests/test_parser.py

@@ -102,6 +102,64 @@ def test_parse_numbered_paragraph(self) -> None:
         assert df.iloc[0]["paragraph"] == "1"
         assert df.iloc[0]["text"] == "First paragraph"
 
+    def test_parse_metadata_propagation_per_article(self) -> None:
+        """Regression: article/group/section must reflect position, not final values."""
+        html = """<html><body>
+            <p class="doc-ti">TEST REGULATION</p>
+            <p class="ti-section-1">GENERAL PROVISIONS</p>
+            <p class="ti-grseq-1">Chapter I Scope</p>
+            <p class="ti-art">Article 1</p>
+            <p class="normal">(1) First article first paragraph.</p>
+            <p class="normal">(2) First article second paragraph.</p>
+            <p class="ti-art">Article 2</p>
+            <p class="normal">(1) Second article first paragraph.</p>
+            <p class="ti-section-2">FINAL PROVISIONS</p>
+            <p class="ti-grseq-2">Chapter II Entry into force</p>
+            <p class="ti-art">Article 3</p>
+            <p class="normal">(1) Third article first paragraph.</p>
+        </body></html>"""
+        df = parse_html(html)
+        assert len(df) == 4
+
+        # Article 1 rows
+        assert df.iloc[0]["article"] == "1"
+        assert df.iloc[0]["section"] == "GENERAL PROVISIONS"
+        assert df.iloc[0]["group"] == "Chapter I Scope"
+        assert df.iloc[0]["paragraph"] == "1"
+        assert df.iloc[1]["article"] == "1"
+        assert df.iloc[1]["paragraph"] == "2"
+
+        # Article 2 row
+        assert df.iloc[2]["article"] == "2"
+        assert df.iloc[2]["section"] == "GENERAL PROVISIONS"
+        assert df.iloc[2]["group"] == "Chapter I Scope"
+        assert df.iloc[2]["paragraph"] == "1"
+
+        # Article 3 row (different section and group)
+        assert df.iloc[3]["article"] == "3"
+        assert df.iloc[3]["section"] == "FINAL PROVISIONS"
+        assert df.iloc[3]["group"] == "Chapter II Entry into force"
+        assert df.iloc[3]["paragraph"] == "1"
+
+    def test_preamble_has_no_article(self) -> None:
+        """Preamble text before any article should not have article metadata."""
+        html = """<html><body>
+            <p class="doc-ti">TEST REGULATION</p>
+            <p class="normal">THE EUROPEAN PARLIAMENT AND THE COUNCIL,</p>
+            <p class="normal">Having regard to the Treaty,</p>
+            <p class="ti-art">Article 1</p>
+            <p class="normal">(1) Article content.</p>
+        </body></html>"""
+        df = parse_html(html)
+        assert len(df) == 3
+
+        # Preamble rows should have no article
+        assert df.iloc[0]["article"] is None
+        assert df.iloc[1]["article"] is None
+
+        # Article 1 row should have article
+        assert df.iloc[2]["article"] == "1"
+
 
 class TestParseCelexId:
     """Tests for CELEX ID parsing and validation."""