fix(file): treat blob fields as opaque single columns in 2.0 projection validation

python/python/tests/test_blob.py

@@ -1668,3 +1668,34 @@ def test_to_pandas_returns_blob_files_when_nested_field_is_aliased(
     assert images[0].readall() == b"foo"
     assert images[1] is None
     assert images[2].readall() == b"baz"
+
+
+@pytest.mark.parametrize("data_storage_version", ["2.0", "2.1"])
+def test_blob_scan_across_file_versions(tmp_path, data_storage_version):
+    """Blob descriptor structs are one opaque column in every file version.
+
+    Regression test for 9.0.0b14: the projection length validator descended
+    into the descriptor's position/size children on 2.0 files, making any
+    blob-bearing 2.0 dataset unreadable ("ran out at field 'position'").
+    """
+    schema = pa.schema(
+        [
+            pa.field("a", pa.int64()),
+            pa.field("b", pa.large_binary(), metadata={"lance-encoding:blob": "true"}),
+        ]
+    )
+    values = [b"x", b"yy", b"zzz"]
+    table = pa.table({"a": [1, 2, 3], "b": values}).cast(schema)
+    ds = lance.write_dataset(
+        table,
+        tmp_path / "blob_scan_versions",
+        data_storage_version=data_storage_version,
+    )
+
+    full = ds.to_table()
+    assert full.num_rows == 3
+    projected = ds.to_table(columns=["b"])
+    assert projected.num_rows == 3
+
+    blobs = ds.take_blobs("b", indices=[0, 1, 2])
+    assert [blob.read() for blob in blobs] == values

rust/lance-file/src/reader.rs

@@ -518,18 +518,21 @@ struct Footer {
 const FOOTER_LEN: usize = 40;
 
 // How a field maps onto physical columns, shared by the projection-building and
-// projection-validation walks so they stay in lockstep. In the 2.0 layout every
-// field (including structs and lists) has its own column; in 2.1 only leaves do,
-// and blob/packed-struct fields are opaque (a single column with no descent).
+// projection-validation walks so they stay in lockstep. Blob and packed-struct
+// fields are opaque in every file version: one column, no descent — their
+// descriptor/packed children never get columns of their own. Otherwise, in the
+// 2.0 layout every field (including structs and lists) has its own column; in
+// 2.1 only leaves do.
 // Returns `(contributes, recurse)`: whether the field has its own column and
 // whether to walk into its children. The DFS order is the field's own column (if
 // any) followed by its children, so a field's root (first) column is always the
 // first entry of its sub-slice.
 fn field_column_shape(field: &Field, is_structural: bool) -> (bool, bool) {
-    let contributes =
-        !is_structural || field.children.is_empty() || field.is_blob() || field.is_packed_struct();
-    let recurse = !is_structural || (!field.is_blob() && !field.is_packed_struct());
-    (contributes, recurse)
+    if field.is_blob() || field.is_packed_struct() {
+        return (true, false);
+    }
+    let contributes = !is_structural || field.children.is_empty();
+    (contributes, true)
 }
 
 // Whether a field's children each cover the same rows as the field itself. Struct
@@ -3884,6 +3887,35 @@ mod tests {
         );
     }
 
+    // A blob column's descriptor struct is one opaque physical column in every
+    // file version. The validator must not descend into `position`/`size`
+    // expecting columns of their own — with the 2.0 layout that made every
+    // blob-bearing dataset unreadable ("ran out at field 'position'").
+    #[rstest]
+    fn test_validate_length_blob_descriptor(#[values(false, true)] is_structural: bool) {
+        let descriptor = DataType::Struct(Fields::from(vec![
+            Field::new("position", DataType::UInt64, true),
+            Field::new("size", DataType::UInt64, true),
+        ]));
+        let blob_field = Field::new("b", descriptor, true)
+            .with_metadata([(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string())].into());
+        let arrow = ArrowSchema::new(vec![blob_field]);
+        let schema = Schema::try_from(&arrow).unwrap();
+        let column_len = |_c: usize| Ok(7u64);
+        let mut cursor = 0usize;
+        let rows = validate_field_length(
+            &schema.fields[0],
+            is_structural,
+            true,
+            &[0],
+            &mut cursor,
+            &column_len,
+        )
+        .unwrap();
+        assert_eq!(rows, 7);
+        assert_eq!(cursor, 1, "blob must consume exactly its one opaque column");
+    }
+
     #[test]
     fn test_validate_length_list_and_empty_struct() {
         let validate = |dt: DataType,