From c3eed842c57b283769a096944b65c2dc571ee3a4 Mon Sep 17 00:00:00 2001 From: Keval Date: Mon, 28 Sep 2026 15:16:30 +0530 Subject: [PATCH 1/2] fix(s3): keep child objects visible beneath folder markers --- crates/openlake_io/src/local_fs.rs | 45 ++++++++++++++------------- crates/openlake_storage/src/engine.rs | 18 +++++++++++ 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/crates/openlake_io/src/local_fs.rs b/crates/openlake_io/src/local_fs.rs index aa69078..972b4ed 100644 --- a/crates/openlake_io/src/local_fs.rs +++ b/crates/openlake_io/src/local_fs.rs @@ -1306,34 +1306,35 @@ fn walk_dir_local_inner<'a>( continue; } - match backend.read_version("", volume, &child, None, false).await { + let descend = match backend.read_version("", volume, &child, None, false).await { Ok(fi) => { - if !prefix_filter.is_empty() && !fi.name.starts_with(prefix_filter) { - continue; + if (prefix_filter.is_empty() || fi.name.starts_with(prefix_filter)) + && start_after.is_none_or(|after| fi.name.as_str() > after) + { + out.push((fi.name.clone(), fi)); } - if let Some(after) = start_after { - if fi.name.as_str() <= after { - continue; - } - } - out.push((fi.name.clone(), fi)); + // An object directory can also contain child objects. + true } Err(IoError::FileNotFound { .. }) => { - if recursive { - walk_dir_local_inner( - backend, - volume, - &child, - prefix_filter, - recursive, - start_after, - max_keys, - out, - ) - .await?; - } + recursive + && std::fs::symlink_metadata(backend.file_path(volume, &child)) + .is_ok_and(|meta| meta.file_type().is_dir()) } Err(_) => continue, + }; + if recursive && descend { + walk_dir_local_inner( + backend, + volume, + &child, + prefix_filter, + recursive, + start_after, + max_keys, + out, + ) + .await?; } } Ok(()) diff --git a/crates/openlake_storage/src/engine.rs b/crates/openlake_storage/src/engine.rs index 8ba71f1..c7ba266 100644 --- a/crates/openlake_storage/src/engine.rs +++ b/crates/openlake_storage/src/engine.rs @@ -3214,6 +3214,24 @@ mod tests { } } + #[compio::test] + async fn list_folder_marker_keeps_child_visible() { + let (_dirs, e) = eng(3, 3).await; + put_bytes(&e, "buk", "folder/child.txt", b"child".to_vec(), None).await; + put_bytes(&e, "buk", "folder/", Vec::new(), None).await; + + let listed = e.list("buk", "folder/", None, 0).await.unwrap(); + let keys: Vec<&str> = listed.iter().map(|object| object.key.as_str()).collect(); + assert_eq!(keys, ["folder/child.txt"]); + + let root = e.list("buk", "", None, 0).await.unwrap(); + assert!(root.iter().any(|object| object.key == "folder/child.txt")); + + let after_marker = e.list("buk", "folder/", Some("folder/"), 1).await.unwrap(); + assert_eq!(after_marker[0].key, "folder/child.txt"); + assert_eq!(get_bytes(&e, "buk", "folder/child.txt").await.1, b"child"); + } + /// 1 MiB payload — straight onto the EC streaming path with the /// default inline cutoff (128 KiB). EC(2+1) on a 3-disk set, no /// faults, exact-bytes round trip via streaming. From bc69887924fe626715e61bd1e43d71136bc8a41a Mon Sep 17 00:00:00 2001 From: Keval Date: Mon, 28 Sep 2026 15:37:43 +0530 Subject: [PATCH 2/2] fix(s3): skip object storage internals during recursive listing --- crates/openlake_io/src/local_fs.rs | 62 ++++++++++++++++++++------- crates/openlake_storage/src/engine.rs | 30 +++++++++++++ 2 files changed, 76 insertions(+), 16 deletions(-) diff --git a/crates/openlake_io/src/local_fs.rs b/crates/openlake_io/src/local_fs.rs index 972b4ed..c7057d0 100644 --- a/crates/openlake_io/src/local_fs.rs +++ b/crates/openlake_io/src/local_fs.rs @@ -962,6 +962,7 @@ impl StorageBackend for LocalFsBackend { recursive, start_after, max_keys, + None, &mut out, ) .await?; @@ -1269,6 +1270,7 @@ fn walk_dir_local_inner<'a>( recursive: bool, start_after: Option<&'a str>, max_keys: Option, + skip_internal: Option>, out: &'a mut Vec<(String, FileInfo)>, ) -> std::pin::Pin> + 'a>> { Box::pin(async move { @@ -1280,6 +1282,13 @@ fn walk_dir_local_inner<'a>( let mut entries = backend.list_dir(volume, dir, 0).await?; entries.sort(); for name in entries { + if skip_internal.as_ref().is_some_and(|data_dirs| { + name == META_FILENAME + || name == META_BACKUP_FILENAME + || data_dirs.iter().any(|data_dir| data_dir == &name) + }) { + continue; + } if let Some(n) = max_keys { if out.len() >= n { return Ok(()); @@ -1306,23 +1315,43 @@ fn walk_dir_local_inner<'a>( continue; } - let descend = match backend.read_version("", volume, &child, None, false).await { - Ok(fi) => { - if (prefix_filter.is_empty() || fi.name.starts_with(prefix_filter)) - && start_after.is_none_or(|after| fi.name.as_str() > after) - { - out.push((fi.name.clone(), fi)); + let (descend, child_skip_internal) = + match compio::fs::read(backend.meta_path(volume, &child)).await { + Ok(bytes) => { + let records = match xl_meta::decode_all(bytes::Bytes::from(bytes)) { + Ok(records) => records, + Err(_) => continue, + }; + let data_dirs = records + .iter() + .filter(|record| !record.data_dir.is_empty()) + .map(|record| record.data_dir.clone()) + .collect(); + let mut fi = xl_meta::file_info_from_record( + records + .into_iter() + .next() + .expect("decoded metadata has a version"), + volume, + &child, + ); + fi.data = None; + if (prefix_filter.is_empty() || fi.name.starts_with(prefix_filter)) + && start_after.is_none_or(|after| fi.name.as_str() > after) + { + out.push((fi.name.clone(), fi)); + } + // Descend into real children, skipping this object's shard directories. + (true, Some(data_dirs)) } - // An object directory can also contain child objects. - true - } - Err(IoError::FileNotFound { .. }) => { - recursive - && std::fs::symlink_metadata(backend.file_path(volume, &child)) - .is_ok_and(|meta| meta.file_type().is_dir()) - } - Err(_) => continue, - }; + Err(error) if error.kind() == std::io::ErrorKind::NotFound => ( + recursive + && std::fs::symlink_metadata(backend.file_path(volume, &child)) + .is_ok_and(|meta| meta.file_type().is_dir()), + None, + ), + Err(_) => continue, + }; if recursive && descend { walk_dir_local_inner( backend, @@ -1332,6 +1361,7 @@ fn walk_dir_local_inner<'a>( recursive, start_after, max_keys, + child_skip_internal, out, ) .await?; diff --git a/crates/openlake_storage/src/engine.rs b/crates/openlake_storage/src/engine.rs index c7ba266..bab259a 100644 --- a/crates/openlake_storage/src/engine.rs +++ b/crates/openlake_storage/src/engine.rs @@ -3232,6 +3232,36 @@ mod tests { assert_eq!(get_bytes(&e, "buk", "folder/child.txt").await.1, b"child"); } + #[compio::test] + async fn list_folder_marker_skips_version_data_directories() { + let (dirs, e) = eng(3, 3).await; + e.put_bucket_versioning("buk", VersioningStatus::Enabled) + .await + .unwrap(); + put_bytes(&e, "buk", "folder/child.txt", b"child".to_vec(), None).await; + put_bytes(&e, "buk", "folder/", vec![1; 200 * 1024], None).await; + std::thread::sleep(std::time::Duration::from_millis(2)); + put_bytes(&e, "buk", "folder/", vec![2; 200 * 1024], None).await; + + for dir in &dirs { + let folder = dir.path().join("buk/folder"); + let records = openlake_io::xl_meta::decode_all(bytes::Bytes::from( + std::fs::read(folder.join("xl.meta")).unwrap(), + )) + .unwrap(); + assert_eq!(records.len(), 2); + for record in records { + let internal = folder.join(record.data_dir).join("sentinel"); + std::fs::create_dir_all(&internal).unwrap(); + std::fs::copy(folder.join("child.txt/xl.meta"), internal.join("xl.meta")).unwrap(); + } + } + + let listed = e.list("buk", "folder/", None, 0).await.unwrap(); + let keys: Vec<&str> = listed.iter().map(|object| object.key.as_str()).collect(); + assert_eq!(keys, ["folder/child.txt"]); + } + /// 1 MiB payload — straight onto the EC streaming path with the /// default inline cutoff (128 KiB). EC(2+1) on a 3-disk set, no /// faults, exact-bytes round trip via streaming.