diff --git a/pyproject.toml b/pyproject.toml index 13b41e5a9f13..098525f7ee6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,7 +200,7 @@ dependencies = [ "braintrust-core>=0.0.59,<1", "wrapt==1.17.3", "limits==5.8.0", - "deltalite==0.1.2", + "deltalite==0.1.3", ] [dependency-groups] diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4afce2c4d3cd..60c47c794d1f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -3788,7 +3788,7 @@ dependencies = [ [[package]] name = "deltalite-python" -version = "0.1.2" +version = "0.1.3" dependencies = [ "arrow", "arrow-array", diff --git a/rust/deltalite/core/src/upsert.rs b/rust/deltalite/core/src/upsert.rs index 7ddef11bb308..d8a6af4df639 100644 --- a/rust/deltalite/core/src/upsert.rs +++ b/rust/deltalite/core/src/upsert.rs @@ -240,6 +240,12 @@ pub struct UpsertStats { pub source_rows: usize, /// Source rows carrying a NULL PK component (always inserted, never matched). pub null_pk_rows: usize, + /// Wall-clock ms spent planning: listing each partition's files and pruning which to rewrite. + pub plan_ms: u64, + /// Wall-clock ms spent rewriting the touched partitions (reading + probing targets, writing files). + pub rewrite_ms: u64, + /// Wall-clock ms spent committing the new file set to the Delta log. + pub commit_ms: u64, } /// A file selected for rewrite, with the metadata needed to tombstone it. @@ -604,6 +610,7 @@ async fn upsert_inner( // Index of the first PK column in the table schema, for stats-based pruning. let pk0_idx = table_schema.index_of(&opts.primary_keys[0]).ok(); + let plan_started = Instant::now(); let mut work = Vec::new(); for (value, source) in groups { let files = list_partition_files(table, partition_col.as_deref(), &value).await?; @@ -628,7 +635,10 @@ async fn upsert_inner( }); } + let plan_ms = plan_started.elapsed().as_millis() as u64; + // --- Rewrite (parallel across partitions) ---------------------------------------- + let rewrite_started = Instant::now(); let partitions_touched = work.len(); let semaphore = Arc::new(Semaphore::new(opts.max_parallel_partitions.max(1))); // Per-call byte budget in KiB units (tokio's acquire_many takes u32); the @@ -696,7 +706,10 @@ async fn upsert_inner( actions.extend(outcome.actions); } + let rewrite_ms = rewrite_started.elapsed().as_millis() as u64; + // --- Commit ---------------------------------------------------------------------- + let commit_started = Instant::now(); let predicate = partition_col.as_ref().map(|c| { let vals: Vec = actions .iter() @@ -730,7 +743,11 @@ async fn upsert_inner( .with_actions(actions) .build(Some(snapshot), table.log_store(), operation) .await?; + let commit_ms = commit_started.elapsed().as_millis() as u64; + stats.plan_ms = plan_ms; + stats.rewrite_ms = rewrite_ms; + stats.commit_ms = commit_ms; stats.version = i64::try_from(finalized.version()) .map_err(|_| Error::Generic("committed version overflows i64".into()))?; tracing::Span::current().record("version", stats.version); @@ -743,6 +760,9 @@ async fn upsert_inner( rows_updated = stats.rows_updated, rows_inserted = stats.rows_inserted, rows_copied = stats.rows_copied, + plan_ms = stats.plan_ms, + rewrite_ms = stats.rewrite_ms, + commit_ms = stats.commit_ms, "upsert committed" ); Ok(stats) diff --git a/rust/deltalite/python/Cargo.toml b/rust/deltalite/python/Cargo.toml index 5e71120f17c3..0e9297287070 100644 --- a/rust/deltalite/python/Cargo.toml +++ b/rust/deltalite/python/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "deltalite-python" # Mirrors pyproject.toml in this directory (always identical); bump both together. -version = "0.1.2" +version = "0.1.3" edition = "2021" # CI compiles with this exact toolchain; cargo's MSRV-aware resolver uses it to # avoid picking dependency versions CI cannot build (aws-* crates move MSRV often). diff --git a/rust/deltalite/python/README.md b/rust/deltalite/python/README.md index 60179085c8f1..2d46b9b5875b 100644 --- a/rust/deltalite/python/README.md +++ b/rust/deltalite/python/README.md @@ -108,9 +108,12 @@ rejected (raising `DeltaLiteError`) rather than silently double-inserted. ### `UpsertStats` -Returned by `upsert`. Fields: `version`, `partitions_touched`, `files_added`, +Returned by `upsert`. Counts: `version`, `partitions_touched`, `files_added`, `files_removed`, `files_carried_over`, `files_probed`, `rows_updated`, -`rows_inserted`, `rows_copied`, `source_rows`, `null_pk_rows`. +`rows_inserted`, `rows_copied`, `source_rows`, `null_pk_rows`. Per-phase +wall-clock timings (milliseconds): `plan_ms` (listing + pruning files), +`rewrite_ms` (reading + rewriting the touched partitions), `commit_ms` +(committing to the Delta log). ### Exceptions diff --git a/rust/deltalite/python/pyproject.toml b/rust/deltalite/python/pyproject.toml index df5fc6807ca0..b4953343d83e 100644 --- a/rust/deltalite/python/pyproject.toml +++ b/rust/deltalite/python/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "maturin" [project] name = "deltalite" # Mirrors `rust/deltalite/python/Cargo.toml` (always identical); bump both together. -version = "0.1.2" +version = "0.1.3" description = "Streaming partition upsert for Delta tables, replacing delta-rs SQL MERGE" readme = "README.md" requires-python = ">=3.12" diff --git a/rust/deltalite/python/src/lib.rs b/rust/deltalite/python/src/lib.rs index a7282aefa38f..af149b6b0b2c 100644 --- a/rust/deltalite/python/src/lib.rs +++ b/rust/deltalite/python/src/lib.rs @@ -161,6 +161,12 @@ pub struct UpsertStats { pub source_rows: usize, #[pyo3(get)] pub null_pk_rows: usize, + #[pyo3(get)] + pub plan_ms: u64, + #[pyo3(get)] + pub rewrite_ms: u64, + #[pyo3(get)] + pub commit_ms: u64, } #[pymethods] @@ -197,6 +203,9 @@ impl From for UpsertStats { rows_copied: s.rows_copied, source_rows: s.source_rows, null_pk_rows: s.null_pk_rows, + plan_ms: s.plan_ms, + rewrite_ms: s.rewrite_ms, + commit_ms: s.commit_ms, } } } diff --git a/uv.lock b/uv.lock index 53d66129ee60..20f1548572a2 100644 --- a/uv.lock +++ b/uv.lock @@ -1731,16 +1731,16 @@ pyarrow = [ [[package]] name = "deltalite" -version = "0.1.2" +version = "0.1.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/1e/49659fed92532267fbfc9298a16504ce8f13a26382e6355a0b43e337dade/deltalite-0.1.2.tar.gz", hash = "sha256:9555b17d7e57dc7e869317db4db772acfe067ad53aed2ac0699e751c401e3f02", size = 172868, upload-time = "2026-07-29T13:57:28.814Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/18/0ebcff958c74ea79f1e8dfbb261f5474bb06033da21ced1bde4d3e1cc2e7/deltalite-0.1.3.tar.gz", hash = "sha256:5daafab4ed9d6f54cdcfea1cb7bfd900b906b6ab9402759c5a176b06fd943815", size = 173464, upload-time = "2026-07-31T13:34:15.853Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/b0/2618124dc3bc8433248ffa3855bae050f6e12f0cc098857e2adc187545e5/deltalite-0.1.2-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:de3506abd5c07df8f270b6ab125a62324eda64b968614d2daf7e35a97d031123", size = 17957003, upload-time = "2026-07-29T13:57:08.139Z" }, - { url = "https://files.pythonhosted.org/packages/a6/d5/a4ca6595ad7efcc1550a39cd4a3fff67a13076db5623582100e26a6dd5c0/deltalite-0.1.2-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:e48f84c81c894ffb394aa92824989b64c017fc1bd4ffe4ba97b4d79d72e06829", size = 17838440, upload-time = "2026-07-29T13:57:10.827Z" }, - { url = "https://files.pythonhosted.org/packages/10/7c/da3966cf9ab0db82285dc35932dde36da665ad22808051c51841bae999f4/deltalite-0.1.2-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1c0a45ce1bcf920170d35f707018d66da9476e1c12adcc8589742c9fdce74537", size = 63133529, upload-time = "2026-07-29T13:57:14.25Z" }, - { url = "https://files.pythonhosted.org/packages/05/60/fa7c805b09f7f8d96da93572f640c3a7a256417d5a4b3a4425a7b46770a2/deltalite-0.1.2-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:fc4af9d59fa944ebd906d9b395f8dcf75ff41a5c5f95540abc90914f417c9851", size = 65444603, upload-time = "2026-07-29T13:57:18.315Z" }, - { url = "https://files.pythonhosted.org/packages/22/9c/52d86d4ffb7d30dc761888b130f217e6a09d37e0ca436d3d18adba9aaed6/deltalite-0.1.2-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1eb6b872730d03367b572f8fdb876dfb08875d0d1d3a06fc35bbd40db42a6aaf", size = 62721871, upload-time = "2026-07-29T13:57:22.439Z" }, - { url = "https://files.pythonhosted.org/packages/01/b2/f7268ff3f7731d2421264ab2b342307893ce7e09659e98b36608ef9b5503/deltalite-0.1.2-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5b7ab6824c16a0a6fa23ce4dca15c3fdc7fbbbd98eb25dd9a290f1809842e524", size = 63793714, upload-time = "2026-07-29T13:57:26.13Z" }, + { url = "https://files.pythonhosted.org/packages/69/b8/42fed73269c9b93aace404b1c33dfa5886cb9d03cd377982d3b70b4808b9/deltalite-0.1.3-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f43539e33e90d9d2d4f6010a38a69ac1ebc943b7ddb34cb07682734b7d284594", size = 17956004, upload-time = "2026-07-31T13:33:58.697Z" }, + { url = "https://files.pythonhosted.org/packages/68/3a/1f9fd47e3df3fc534abb43e97898c03c7e813f566ba484b5e3f3df43fc35/deltalite-0.1.3-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:91a5b5f72e37701ac1d4541681be99a78dd14456fadcbd685026c8d91be1aa41", size = 17837576, upload-time = "2026-07-31T13:34:00.907Z" }, + { url = "https://files.pythonhosted.org/packages/31/f8/e10658fdb57819b473c824618928b659ad65f541143e9c7a9653a93e9084/deltalite-0.1.3-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4e7cc11b98f4211027f9ab387fc01b6592430b4d699422bac6f5f1a8774e8210", size = 63147119, upload-time = "2026-07-31T13:34:03.7Z" }, + { url = "https://files.pythonhosted.org/packages/d6/11/5aea109ae137d97e19f786e6e78114b326e8f334a71c534193815b69b006/deltalite-0.1.3-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ed2d864ab2b63ffd83589f4fc24b23b69ccfc0165e593f04121e7da89b2c4e37", size = 65438695, upload-time = "2026-07-31T13:34:07.314Z" }, + { url = "https://files.pythonhosted.org/packages/29/6b/2a5ea282e1ded2dfdc97f2dccc9fce88ec419a15f366db229a5e106160df/deltalite-0.1.3-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:206e41394a61182327120351e19df16932481a67a4edb14b3e77cbd6afd8e07e", size = 62731281, upload-time = "2026-07-31T13:34:10.303Z" }, + { url = "https://files.pythonhosted.org/packages/2c/08/f0ddae42c7aef923a2b4b43b797b6a7ab781f05e05bfeee3916a3106a969/deltalite-0.1.3-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5f4a13be6ff584cb18b2162877fcb9f017f5d7a650039c271679c33d89914b95", size = 63797093, upload-time = "2026-07-31T13:34:13.349Z" }, ] [[package]] @@ -4661,7 +4661,7 @@ name = "nvidia-cudnn-cu12" version = "9.10.2.21" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "platform_machine != 's390x' and sys_platform != 'emscripten'" }, + { name = "nvidia-cublas-cu12" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, @@ -4672,7 +4672,7 @@ name = "nvidia-cufft-cu12" version = "11.3.3.83" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine != 's390x' and sys_platform != 'emscripten'" }, + { name = "nvidia-nvjitlink-cu12" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, @@ -4699,9 +4699,9 @@ name = "nvidia-cusolver-cu12" version = "11.7.3.90" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "platform_machine != 's390x' and sys_platform != 'emscripten'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine != 's390x' and sys_platform != 'emscripten'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine != 's390x' and sys_platform != 'emscripten'" }, + { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cusparse-cu12" }, + { name = "nvidia-nvjitlink-cu12" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, @@ -4712,7 +4712,7 @@ name = "nvidia-cusparse-cu12" version = "12.5.8.93" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine != 's390x' and sys_platform != 'emscripten'" }, + { name = "nvidia-nvjitlink-cu12" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, @@ -5371,7 +5371,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "sys_platform != 'emscripten'" }, + { name = "ptyprocess" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -5859,7 +5859,7 @@ requires-dist = [ { name = "databricks-sql-connector", specifier = "~=4.3.0" }, { name = "defusedxml", specifier = "~=0.7.1" }, { name = "deltalake", extras = ["pyarrow"], specifier = "==1.6.1" }, - { name = "deltalite", specifier = "==0.1.2" }, + { name = "deltalite", specifier = "==0.1.3" }, { name = "disposable-email-domains", specifier = "~=0.0.140" }, { name = "dj-database-url", specifier = "~=3.0.0" }, { name = "django", specifier = "~=5.2.0" }, @@ -7642,7 +7642,7 @@ name = "shadowcopy" version = "0.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wmi", marker = "sys_platform != 'emscripten'" }, + { name = "wmi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/12/44/c00420a3b7bcdf830529b933392b566c876a4944a24f31e126eb6fd28647/shadowcopy-0.0.4.tar.gz", hash = "sha256:ed89817dda065f893607a04c0b7d6b3b34c3507a4711f441111a4bcb1b1826c0", size = 4138, upload-time = "2023-07-08T00:01:35.266Z" } wheels = [ @@ -8076,7 +8076,7 @@ name = "tbb" version = "2022.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "tcmlib", marker = "platform_machine != 's390x' and sys_platform != 'emscripten'" }, + { name = "tcmlib" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/08/59/8d381a2cfe8d36c4f4ff9f94769ff2809bfc16014d888360b0e24c7e5c6b/tbb-2022.3.1-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:534c089eca6bcf148408684c156229d5f09c01d323b685b15739f41985b529d7", size = 4178630, upload-time = "2026-01-22T05:30:20.589Z" }, @@ -8458,7 +8458,7 @@ name = "triton" version = "3.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "setuptools", marker = "platform_machine != 's390x' and sys_platform != 'emscripten'" }, + { name = "setuptools" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/30/7b/0a685684ed5322d2af0bddefed7906674f67974aa88b0fae6e82e3b766f6/triton-3.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00be2964616f4c619193cb0d1b29a99bd4b001d7dc333816073f92cf2a8ccdeb", size = 155569223, upload-time = "2025-07-30T19:58:44.017Z" }, @@ -9165,7 +9165,7 @@ name = "wmi" version = "1.5.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywin32", marker = "sys_platform != 'emscripten'" }, + { name = "pywin32" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d4/66/6364deb0a03415f96c66803d8c4379f808f2401da3bdb183348487b10510/WMI-1.5.1.tar.gz", hash = "sha256:b6a6be5711b1b6c8d55bda7a8befd75c48c12b770b9d227d31c1737dbf0d40a6", size = 26254, upload-time = "2020-04-28T08:22:58.096Z" } wheels = [