From c8659993a1c35e8590dcdbe862a374d65de765b1 Mon Sep 17 00:00:00 2001 From: Mo Date: Tue, 10 Feb 2026 11:28:01 +0100 Subject: [PATCH] bugfix: metadata parsing --- CHANGELOG.md | 6 ++++ pyproject.toml | 2 +- src/eurlxp/parser.py | 75 +++++++++++++++++++++----------------------- tests/test_parser.py | 58 ++++++++++++++++++++++++++++++++++ uv.lock | 2 +- 5 files changed, 101 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1013473..a29abaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.1] - 2026-02-10 + +### Fixed + +- **`parse_html` metadata propagation** - Fixed bug where `article`, `group`, and `section` columns were assigned the same values (from the last element in the document) for all rows. The parser now processes `

` tags in a single pass in document order, so each row reflects its actual structural position. Preamble/recital rows before any article now correctly have `None` for `article`. ([#1](https://github.com/morrieinmaas/eurlxp/issues/1)) + ## [0.4.0] - 2026-02-04 ### Added diff --git a/pyproject.toml b/pyproject.toml index fb65c63..0e75235 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "eurlxp" -version = "0.4.0" +version = "0.4.1" description = "A modern EUR-Lex parser for Python - fetch and parse EU legal documents" readme = "README.md" license = { text = "MIT" } diff --git a/src/eurlxp/parser.py b/src/eurlxp/parser.py index f67d551..d4d08b4 100644 --- a/src/eurlxp/parser.py +++ b/src/eurlxp/parser.py @@ -205,6 +205,9 @@ def _parse_html_with_beautifulsoup(html: str) -> list[ParseResult]: Tries lxml-xml parser first (for XHTML documents), then falls back to lxml HTML parser for older HTML documents. + + Processes all

tags in a single pass in document order so that + article, group, and section context updates apply only to subsequent rows. """ from bs4 import BeautifulSoup @@ -218,55 +221,47 @@ def _parse_html_with_beautifulsoup(html: str) -> list[ParseResult]: results: list[ParseResult] = [] context = ParseContext() - # Find document title - # Supports: OJ format (doc-ti, oj-doc-ti), Commission format (Titreobjet, Typedudocument) - for doc_ti in soup.find_all("p", class_=["doc-ti", "oj-doc-ti", "Titreobjet", "Typedudocument"]): - text = doc_ti.get_text(strip=True) - if text: + doc_title_classes = {"doc-ti", "oj-doc-ti", "Titreobjet", "Typedudocument"} + article_title_classes = {"ti-art", "oj-ti-art", "Titrearticle"} + text_classes = {"normal", "oj-normal", "Normal"} + + # Single pass through all

tags in document order + for p_tag in soup.find_all("p"): + css_classes = p_tag.get("class") or [] + if isinstance(css_classes, str): + css_classes = [css_classes] + css_class_str = " ".join(css_classes) + css_class_set = set(css_classes) + + text = p_tag.get_text(strip=True) + if not text: + continue + + # Document title + if css_class_set & doc_title_classes: if context.document is None: context.document = "" context.document += text results.append(ParseResult(text=text, item_type="doc-title", ref=[], context=context.copy())) - # Find article titles - # Supports: OJ format (ti-art, oj-ti-art), Commission format (Titrearticle) - for ti_art in soup.find_all("p", class_=["ti-art", "oj-ti-art", "Titrearticle"]): - text = ti_art.get_text(strip=True) - if text: + # Article title + elif css_class_set & article_title_classes: context.article = text.replace("Article", "").strip() + context.paragraph = None results.append(ParseResult(text=text, item_type="art-title", ref=[], context=context.copy())) - # Find group titles (ti-grseq-* classes) - for p_tag in soup.find_all("p"): - css_class = p_tag.get("class") - if css_class is None: - css_class = "" - elif isinstance(css_class, list): - css_class = " ".join(css_class) - if css_class and ("ti-grseq-" in css_class or "oj-ti-grseq-" in css_class): - text = p_tag.get_text(strip=True) - if text: - context.group = text - results.append(ParseResult(text=text, item_type="group-title", ref=[], context=context.copy())) + # Group title (ti-grseq-* or oj-ti-grseq-* classes) + elif "ti-grseq-" in css_class_str or "oj-ti-grseq-" in css_class_str: + context.group = text + results.append(ParseResult(text=text, item_type="group-title", ref=[], context=context.copy())) - # Find section titles (ti-section-* classes) - for p_tag in soup.find_all("p"): - css_class = p_tag.get("class") - if css_class is None: - css_class = "" - elif isinstance(css_class, list): - css_class = " ".join(css_class) - if css_class and ("ti-section-" in css_class or "oj-ti-section-" in css_class): - text = p_tag.get_text(strip=True) - if text: - context.section = text - results.append(ParseResult(text=text, item_type="section-title", ref=[], context=context.copy())) - - # Find normal text paragraphs - # Supports: OJ format (normal, oj-normal), Commission format (Normal - capital N) - for normal in soup.find_all("p", class_=["normal", "oj-normal", "Normal"]): - text = normal.get_text(strip=True) - if text: + # Section title (ti-section-* or oj-ti-section-* classes) + elif "ti-section-" in css_class_str or "oj-ti-section-" in css_class_str: + context.section = text + results.append(ParseResult(text=text, item_type="section-title", ref=[], context=context.copy())) + + # Normal text paragraphs + elif css_class_set & text_classes: # Check for numbered paragraphs match = re.match(r"^[(]?([0-9]+)[).]?", text) if match: diff --git a/tests/test_parser.py b/tests/test_parser.py index 7cff76c..d4a1012 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -102,6 +102,64 @@ def test_parse_numbered_paragraph(self) -> None: assert df.iloc[0]["paragraph"] == "1" assert df.iloc[0]["text"] == "First paragraph" + def test_parse_metadata_propagation_per_article(self) -> None: + """Regression: article/group/section must reflect position, not final values.""" + html = """ +

TEST REGULATION

+

GENERAL PROVISIONS

+

Chapter I Scope

+

Article 1

+

(1) First article first paragraph.

+

(2) First article second paragraph.

+

Article 2

+

(1) Second article first paragraph.

+

FINAL PROVISIONS

+

Chapter II Entry into force

+

Article 3

+

(1) Third article first paragraph.

+ """ + df = parse_html(html) + assert len(df) == 4 + + # Article 1 rows + assert df.iloc[0]["article"] == "1" + assert df.iloc[0]["section"] == "GENERAL PROVISIONS" + assert df.iloc[0]["group"] == "Chapter I Scope" + assert df.iloc[0]["paragraph"] == "1" + assert df.iloc[1]["article"] == "1" + assert df.iloc[1]["paragraph"] == "2" + + # Article 2 row + assert df.iloc[2]["article"] == "2" + assert df.iloc[2]["section"] == "GENERAL PROVISIONS" + assert df.iloc[2]["group"] == "Chapter I Scope" + assert df.iloc[2]["paragraph"] == "1" + + # Article 3 row (different section and group) + assert df.iloc[3]["article"] == "3" + assert df.iloc[3]["section"] == "FINAL PROVISIONS" + assert df.iloc[3]["group"] == "Chapter II Entry into force" + assert df.iloc[3]["paragraph"] == "1" + + def test_preamble_has_no_article(self) -> None: + """Preamble text before any article should not have article metadata.""" + html = """ +

TEST REGULATION

+

THE EUROPEAN PARLIAMENT AND THE COUNCIL,

+

Having regard to the Treaty,

+

Article 1

+

(1) Article content.

+ """ + df = parse_html(html) + assert len(df) == 3 + + # Preamble rows should have no article + assert df.iloc[0]["article"] is None + assert df.iloc[1]["article"] is None + + # Article 1 row should have article + assert df.iloc[2]["article"] == "1" + class TestParseCelexId: """Tests for CELEX ID parsing and validation.""" diff --git a/uv.lock b/uv.lock index eedc723..5b4c985 100644 --- a/uv.lock +++ b/uv.lock @@ -210,7 +210,7 @@ wheels = [ [[package]] name = "eurlxp" -version = "0.4.0" +version = "0.4.1" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" },