diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 94da54e2577..fac97fab4a1 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -1668,3 +1668,34 @@ def test_to_pandas_returns_blob_files_when_nested_field_is_aliased( assert images[0].readall() == b"foo" assert images[1] is None assert images[2].readall() == b"baz" + + +@pytest.mark.parametrize("data_storage_version", ["2.0", "2.1"]) +def test_blob_scan_across_file_versions(tmp_path, data_storage_version): + """Blob descriptor structs are one opaque column in every file version. + + Regression test for 9.0.0b14: the projection length validator descended + into the descriptor's position/size children on 2.0 files, making any + blob-bearing 2.0 dataset unreadable ("ran out at field 'position'"). + """ + schema = pa.schema( + [ + pa.field("a", pa.int64()), + pa.field("b", pa.large_binary(), metadata={"lance-encoding:blob": "true"}), + ] + ) + values = [b"x", b"yy", b"zzz"] + table = pa.table({"a": [1, 2, 3], "b": values}).cast(schema) + ds = lance.write_dataset( + table, + tmp_path / "blob_scan_versions", + data_storage_version=data_storage_version, + ) + + full = ds.to_table() + assert full.num_rows == 3 + projected = ds.to_table(columns=["b"]) + assert projected.num_rows == 3 + + blobs = ds.take_blobs("b", indices=[0, 1, 2]) + assert [blob.read() for blob in blobs] == values diff --git a/rust/lance-file/src/reader.rs b/rust/lance-file/src/reader.rs index 0fc0baf0ee0..7bdba62b76f 100644 --- a/rust/lance-file/src/reader.rs +++ b/rust/lance-file/src/reader.rs @@ -518,18 +518,21 @@ struct Footer { const FOOTER_LEN: usize = 40; // How a field maps onto physical columns, shared by the projection-building and -// projection-validation walks so they stay in lockstep. In the 2.0 layout every -// field (including structs and lists) has its own column; in 2.1 only leaves do, -// and blob/packed-struct fields are opaque (a single column with no descent). +// projection-validation walks so they stay in lockstep. Blob and packed-struct +// fields are opaque in every file version: one column, no descent — their +// descriptor/packed children never get columns of their own. Otherwise, in the +// 2.0 layout every field (including structs and lists) has its own column; in +// 2.1 only leaves do. // Returns `(contributes, recurse)`: whether the field has its own column and // whether to walk into its children. The DFS order is the field's own column (if // any) followed by its children, so a field's root (first) column is always the // first entry of its sub-slice. fn field_column_shape(field: &Field, is_structural: bool) -> (bool, bool) { - let contributes = - !is_structural || field.children.is_empty() || field.is_blob() || field.is_packed_struct(); - let recurse = !is_structural || (!field.is_blob() && !field.is_packed_struct()); - (contributes, recurse) + if field.is_blob() || field.is_packed_struct() { + return (true, false); + } + let contributes = !is_structural || field.children.is_empty(); + (contributes, true) } // Whether a field's children each cover the same rows as the field itself. Struct @@ -3884,6 +3887,35 @@ mod tests { ); } + // A blob column's descriptor struct is one opaque physical column in every + // file version. The validator must not descend into `position`/`size` + // expecting columns of their own — with the 2.0 layout that made every + // blob-bearing dataset unreadable ("ran out at field 'position'"). + #[rstest] + fn test_validate_length_blob_descriptor(#[values(false, true)] is_structural: bool) { + let descriptor = DataType::Struct(Fields::from(vec![ + Field::new("position", DataType::UInt64, true), + Field::new("size", DataType::UInt64, true), + ])); + let blob_field = Field::new("b", descriptor, true) + .with_metadata([(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string())].into()); + let arrow = ArrowSchema::new(vec![blob_field]); + let schema = Schema::try_from(&arrow).unwrap(); + let column_len = |_c: usize| Ok(7u64); + let mut cursor = 0usize; + let rows = validate_field_length( + &schema.fields[0], + is_structural, + true, + &[0], + &mut cursor, + &column_len, + ) + .unwrap(); + assert_eq!(rows, 7); + assert_eq!(cursor, 1, "blob must consume exactly its one opaque column"); + } + #[test] fn test_validate_length_list_and_empty_struct() { let validate = |dt: DataType,