diff --git a/CHANGELOG.md b/CHANGELOG.md index 8296232..9691345 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ the project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html ## [Unreleased] +## [0.5.1] — 2026-09-04 + +### Fixed + +- **Numeric singleton rows were folded into the preceding row during + multi-page stitching** (`merger.py`). Sparse numeric rows such as financial + subtotals and fair-value-only lines are now preserved as independent rows; + text-only wrapped-cell continuations still fold as before. +- **Headerless fragments could duplicate their first numeric row during + Docling injection** (`adapters/docling.py`). When extraction demotes an + upstream reader's erroneous `column_header` flag, injection now emits that + row exactly once as ordinary data instead of preserving it as a header and + repeating it in the body. + ## [0.5.0] — 2026-08-23 ### Fixed @@ -242,7 +256,10 @@ the project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html Initial release. -[Unreleased]: https://github.com/pebbleroad/table-stitcher/compare/v0.3.0...HEAD +[Unreleased]: https://github.com/pebbleroad/table-stitcher/compare/v0.5.1...HEAD +[0.5.1]: https://github.com/pebbleroad/table-stitcher/releases/tag/v0.5.1 +[0.5.0]: https://github.com/pebbleroad/table-stitcher/releases/tag/v0.5.0 +[0.4.0]: https://github.com/pebbleroad/table-stitcher/releases/tag/v0.4.0 [0.3.0]: https://github.com/pebbleroad/table-stitcher/releases/tag/v0.3.0 [0.2.0]: https://github.com/pebbleroad/table-stitcher/releases/tag/v0.2.0 [0.1.0]: https://github.com/pebbleroad/table-stitcher/releases/tag/v0.1.0 diff --git a/pyproject.toml b/pyproject.toml index ce8fd03..26f36c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "table-stitcher" -version = "0.5.0" +version = "0.5.1" description = "Reassemble tables split across page boundaries in PDF extraction" readme = "README.md" license = "MIT" diff --git a/src/table_stitcher/adapters/docling.py b/src/table_stitcher/adapters/docling.py index 278a9c6..186e602 100644 --- a/src/table_stitcher/adapters/docling.py +++ b/src/table_stitcher/adapters/docling.py @@ -397,6 +397,20 @@ def _grid_to_dataframe(table: Any, doc: Any) -> pd.DataFrame: df.attrs["pre_header_rows"] = pre_header_rows df.attrs["is_headerless"] = is_headerless + first_content_grid_row = next( + ( + row + for row in grid + if row and any(str(getattr(cell, "text", "") or "").strip() for cell in row if cell) + ), + [], + ) + df.attrs["demoted_numeric_header"] = bool( + is_headerless + and non_empty_cells + and all(_looks_like_data(cell) for cell in non_empty_cells) + and any(getattr(cell, "column_header", False) for cell in first_content_grid_row if cell) + ) return df @@ -563,6 +577,7 @@ def _dataframe_to_docling_data( member_data: Optional[list[Optional[TableData]]] = None, member_pages: Optional[list[Optional[int]]] = None, row_pages_out: Optional[dict[int, int]] = None, + demoted_numeric_header: bool = False, ) -> TableData: """ Converts a pandas DataFrame back into Docling's TableData structure. @@ -586,7 +601,7 @@ def _dataframe_to_docling_data( (``member_pages[0]``); re-emitted body rows map to their source fragment's page; flat-rebuilt rows get no entry (see ``LogicalTable.row_pages``). """ - if df.empty: + if df.empty and not demoted_numeric_header: cols = list(df.columns) if len(df.columns) > 0 else ["Column_0"] header_cells = [] for j, col_name in enumerate(cols): @@ -610,11 +625,21 @@ def _dataframe_to_docling_data( ) # --- Try to reuse original header rows (preserves rowspan/colspan) --- - orig_header_rows, orig_header_cells = _extract_original_header_rows(original_data) + # When extraction classified the anchor as headerless, an upstream reader + # merely misflagged its first data row as a header. Do not preserve that + # flag or synthesize generic DataFrame-column headers during injection. + if demoted_numeric_header: + orig_header_rows, orig_header_cells = [], [] + else: + orig_header_rows, orig_header_cells = _extract_original_header_rows(original_data) num_cols = len(df.columns) - if orig_header_rows: + if demoted_numeric_header: + num_header_rows = 0 + grid = [] + table_cells = [] + elif orig_header_rows: # Use original header rows as-is num_header_rows = len(orig_header_rows) grid: list[list[TableCell]] = list(orig_header_rows) @@ -950,6 +975,7 @@ def extract(self, doc: DoclingDocument, cfg: MultiPageConfig) -> list[TableMeta] continuation_content = [] pre_header_rows = df.attrs.get("pre_header_rows", []) is_headerless = df.attrs.get("is_headerless", False) + demoted_numeric_header = df.attrs.get("demoted_numeric_header", False) if pre_header_rows: for row in pre_header_rows: @@ -1008,6 +1034,7 @@ def extract(self, doc: DoclingDocument, cfg: MultiPageConfig) -> list[TableMeta] continuation_content=continuation_content, is_headerless=is_headerless, content_before=content_before_map.get(idx), + demoted_numeric_header=demoted_numeric_header, ) ) @@ -1110,6 +1137,7 @@ def restore_snapshots(): member_data=member_data, member_pages=member_pages, row_pages_out=row_pages, + demoted_numeric_header=lt.demoted_numeric_header, ) lt.row_pages = row_pages diff --git a/src/table_stitcher/merger.py b/src/table_stitcher/merger.py index 0a5c29b..d63aaf9 100644 --- a/src/table_stitcher/merger.py +++ b/src/table_stitcher/merger.py @@ -169,6 +169,26 @@ def is_empty_value(val: Any) -> bool: return False +_NUMERIC_ONLY_RE = re.compile( + r"""^\s* + [+-]?\s* + (?: + \(\s*[$€£¥]?\s*\d[\d,\s]*(?:\.\d+)?\s*\) + | + [$€£¥]?\s*\d[\d,\s]*(?:\.\d+)?\s*%? + ) + \s*$""", + re.VERBOSE, +) + + +def _is_numeric_only_value(val: Any) -> bool: + """True when ``val`` is a standalone formatted number, not wrapped text.""" + if is_empty_value(val) or isinstance(val, bool): + return False + return bool(_NUMERIC_ONLY_RE.fullmatch(str(val))) + + def clean_malformed_header(col: str) -> str: """Fix headers like 'Name.Name' -> 'Name'.""" col = str(col).strip() @@ -412,6 +432,14 @@ def stitch_split_cells(df: pd.DataFrame, separator: str = "\n") -> pd.DataFrame: cont_idx = nonempty_idxs[0] cont_val = str(next_row_vals[cont_idx]).strip() + + # A standalone number is a legitimate row in financial and + # statistical tables (often a subtotal or fair-value-only line), + # even when every other cell is blank. Folding it into the row + # above silently changes both the row count and the value. + if _is_numeric_only_value(cont_val): + break + target_idx = cont_idx is_url = "://" in cont_val or cont_val.lower().startswith("http") @@ -981,6 +1009,7 @@ def _build_logical_tables(state: _MergeState, cfg: MultiPageConfig) -> list[Logi merge_reason="+".join(merge_reasons), merge_traces=group_traces, warnings=group_warnings, + demoted_numeric_header=state.meta_by_idx[members[0]].demoted_numeric_header, ) ) diff --git a/src/table_stitcher/models.py b/src/table_stitcher/models.py index 874c8c3..e547187 100644 --- a/src/table_stitcher/models.py +++ b/src/table_stitcher/models.py @@ -138,6 +138,8 @@ class TableMeta: the adapter could not place the table in reading order (e.g. an orphan table), in which case the intervening-content guard is skipped for it. """ + demoted_numeric_header: bool = False + """Whether an upstream numeric header row was reclassified as data.""" @dataclass @@ -175,3 +177,9 @@ class LogicalTable: preserved from the anchor map to the anchor's page. Populated during injection; empty for tables that were not merged. """ + demoted_numeric_header: bool = False + """ + Whether the anchor fragment's numeric first grid row was classified as data + rather than a real header. Adapters use this to avoid re-emitting an + upstream header flag that the extraction phase deliberately demoted. + """ diff --git a/tests/integration/fixtures/orphan-pair/varicose-veins-new-table-header-7pg.pt2.expected.yaml b/tests/integration/fixtures/orphan-pair/varicose-veins-new-table-header-7pg.pt2.expected.yaml index 62e0bb6..f60288b 100644 --- a/tests/integration/fixtures/orphan-pair/varicose-veins-new-table-header-7pg.pt2.expected.yaml +++ b/tests/integration/fixtures/orphan-pair/varicose-veins-new-table-header-7pg.pt2.expected.yaml @@ -41,7 +41,7 @@ logical_tables: - 6 - 7 shape: - - 101 + - 103 - 5 columns: - Gender diff --git a/tests/test_docling_adapter.py b/tests/test_docling_adapter.py index 841d66b..49a38a3 100644 --- a/tests/test_docling_adapter.py +++ b/tests/test_docling_adapter.py @@ -718,6 +718,47 @@ def test_merged_table_data_replaced(self): # Data rows come from the merged DataFrame assert doc.tables[0].data.grid[1][0].text == "Alice" + def test_headerless_anchor_does_not_reemit_misflagged_numeric_header(self): + """A demoted upstream header row is emitted once, as ordinary data.""" + doc = _build_doc_with_tables(2) + numeric_row = [ + TableCell( + text=value, + row_span=1, + col_span=1, + column_header=True, + row_header=False, + start_row_offset_idx=0, + end_row_offset_idx=1, + start_col_offset_idx=col, + end_col_offset_idx=col + 1, + ) + for col, value in enumerate(["340,000", "16,746,742", "13,066,200"]) + ] + doc.tables[0].data = TableData( + num_rows=1, + num_cols=3, + table_cells=numeric_row, + grid=[numeric_row], + ) + merged_df = pd.DataFrame( + [ + ["340,000", "16,746,742", "13,066,200"], + ["", "", "29,581,200"], + ], + columns=["Column_0", "Column_1", "Column_2"], + ) + + DoclingAdapter().inject( + doc, + [LogicalTable(0, [0, 1], [4, 5], merged_df, demoted_numeric_header=True)], + ) + + grid = doc.tables[0].data.grid + assert doc.tables[0].data.num_rows == 2 + assert [[cell.text for cell in row] for row in grid] == merged_df.values.tolist() + assert not any(cell.column_header for row in grid for cell in row) + def test_satellite_refs_pruned_from_body(self): """Satellite table references should be removed from doc.body.children.""" doc = _build_doc_with_tables(3) @@ -955,6 +996,24 @@ def _mk_table(rows): class TestHeaderlessDetection: + def test_numeric_reader_header_is_marked_as_demoted_data(self): + grid = [ + [ + SimpleNamespace(text=value, column_header=True) + for value in ["340,000", "16,746,742", "13,066,200"] + ], + [ + SimpleNamespace(text="", column_header=False), + SimpleNamespace(text="", column_header=False), + SimpleNamespace(text="29,581,200", column_header=False), + ], + ] + + df = _grid_to_dataframe(SimpleNamespace(data=SimpleNamespace(grid=grid)), doc=None) + + assert df.attrs["is_headerless"] is True + assert df.attrs["demoted_numeric_header"] is True + def test_comma_separated_decimal_flags_headerless(self): # Retirement-portfolio pattern: first row is data but one cell is a # comma-grouped dollar amount. diff --git a/tests/test_merger.py b/tests/test_merger.py index b40ddb7..cd3005b 100644 --- a/tests/test_merger.py +++ b/tests/test_merger.py @@ -729,6 +729,23 @@ def test_continuation_with_empty_col0_still_folds(self): assert out.shape == (1, 3) assert "rabbit hole" in out.iloc[0, 1] + @pytest.mark.parametrize("subtotal", ["29,581,200", 713044, "$1,234.50", "(9,876)"]) + def test_numeric_singleton_row_is_not_folded(self, subtotal): + """Sparse numeric rows are values/subtotals, not wrapped cell text.""" + df = pd.DataFrame( + [ + ["340,000", "16,746,742", "13,066,200"], + ["", "", subtotal], + ], + columns=["Shares", "Cost", "Fair value"], + ) + + out = stitch_split_cells(df) + + assert out.shape == (2, 3) + assert out.iloc[0, 2] == "13,066,200" + assert out.iloc[1, 2] == subtotal + def test_single_nonempty_cell_folds_into_previous_row(self): df = pd.DataFrame( [