diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..1a0c63b3 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# Beacon pre-commit hook: reject commits containing unformatted Rust code. +# +# Mirrors the `fmt` job in .github/workflows/ci.yml so formatting problems are +# caught locally instead of failing CI. Install it with `make hooks` (which +# points core.hooksPath at .githooks/). +# +# Bypass for a single commit with `git commit --no-verify` if you must. + +set -euo pipefail + +# Nothing to check if no Rust files are staged. +if ! git diff --cached --name-only --diff-filter=ACMR | grep -q '\.rs$'; then + exit 0 +fi + +if ! command -v cargo >/dev/null 2>&1; then + echo "pre-commit: cargo not found on PATH; skipping rustfmt check." >&2 + exit 0 +fi + +if ! cargo fmt -- --check >/dev/null 2>&1; then + echo >&2 + echo "pre-commit: Rust code is not formatted." >&2 + echo " Run 'cargo fmt' and stage the changes, then commit again." >&2 + echo " (Bypass with 'git commit --no-verify' if you really need to.)" >&2 + echo >&2 + exit 1 +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bcdddcee..2b6a9679 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,29 @@ permissions: contents: read jobs: + fmt: + name: Rustfmt + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Rust (1.91) + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.91" + components: rustfmt + + - name: Check formatting + # Plain `cargo fmt` (no --all) formats every workspace member but not + # their path dependencies, so the vendored beacon-binary-format + # submodule (excluded from the workspace) is skipped. It is formatted + # in its own upstream repository. + run: cargo fmt -- --check + ci: name: Clippy, test, and build runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..9ac48f7c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,96 @@ +# Contributing to Beacon + +Thanks for your interest in improving Beacon! This guide covers how to get a +local build going and the checks your pull request needs to pass. + +Issues and pull requests are welcome on +[GitHub](https://github.com/maris-development/beacon/issues). For larger +changes, please open an issue first so we can discuss the approach. + +## Prerequisites + +Beacon is a Rust workspace pinned to the toolchain in +[`rust-toolchain`](rust-toolchain) (currently **1.91**); `rustup` picks it up +automatically. + +The repository uses a git submodule for the binary format crate, so clone +recursively (or initialise it after cloning): + +```bash +git clone --recursive https://github.com/maris-development/beacon.git +# or, in an existing checkout: +git submodule update --init --recursive +``` + +Some crates link against system libraries. On Debian/Ubuntu the build +dependencies are: + +```bash +sudo apt-get install -y \ + build-essential capnproto cmake curl libclang-dev libhdf5-dev \ + libnetcdf-dev libsqlite3-dev netcdf-bin protobuf-compiler sqlite3 +``` + +## Build and test + +```bash +cargo build --workspace +cargo test --workspace +``` + +The admin web UI lives under `clients/` and has its own build; see the +[`Makefile`](Makefile) (`make help`) for convenience targets such as +`make run` (serve the API + UI) and `make dev-ui` (Vite hot-reload). + +## Code style and checks + +CI runs three gates on every push and pull request +([`.github/workflows/ci.yml`](.github/workflows/ci.yml)); run them locally +before pushing. + +### Formatting + +All Rust code must be formatted with `rustfmt`. Beacon uses the **default Rust +style** (pinned in [`rustfmt.toml`](rustfmt.toml)), which matches +[Google's Rust style guide](https://google.github.io/styleguide/rust/) — the +guide mandates rustfmt defaults rather than a custom profile. + +```bash +cargo fmt # format every workspace crate (or: make fmt) +cargo fmt -- --check # what CI enforces; fails on unformatted code +``` + +Plain `cargo fmt` (rather than `cargo fmt --all`) formats the workspace +members but not their path dependencies, so the vendored +`beacon-binary-format` submodule — which lives in its own repository and is +excluded from the workspace — is left alone. + +To catch formatting problems before you commit, install the pre-commit hook +once per clone: + +```bash +make hooks # sets core.hooksPath to .githooks/ +``` + +The hook (`.githooks/pre-commit`) runs the same `cargo fmt -- --check` +whenever a commit touches Rust files and rejects unformatted changes. Bypass it +for a single commit with `git commit --no-verify` if you must. + +### Lints and tests + +```bash +cargo clippy --workspace --lib --bins --tests +cargo test --workspace --no-fail-fast --lib --bins --tests +``` + +## Pull requests + +- Keep the branch focused; unrelated changes are easier to review separately. +- Make sure `cargo fmt -- --check`, `cargo clippy`, and `cargo test` + pass — these are the same checks CI will run. +- Reference any related issue in the PR description. + +## License + +By contributing, you agree that your contributions are licensed under the +project's [AGPL-3.0 license](LICENSE). diff --git a/Cargo.lock b/Cargo.lock index f45a0405..c1b2cb06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,7 +202,7 @@ dependencies = [ "lz4_flex 0.11.6", "moka", "ndarray 0.17.2", - "object_store 0.13.2", + "object_store", "rkyv 0.8.10", "tempfile", "thiserror 2.0.18", @@ -223,95 +223,25 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "arrow" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb98341a7e051bb79731ecb33ec00cbd6e0e315a542d6732b46d462c9215ea2" -dependencies = [ - "arrow-arith 56.2.1", - "arrow-array 56.2.1", - "arrow-buffer 56.2.1", - "arrow-cast 56.2.1", - "arrow-csv 56.2.1", - "arrow-data 56.2.1", - "arrow-ipc 56.2.1", - "arrow-json 56.2.1", - "arrow-ord 56.2.1", - "arrow-row 56.2.1", - "arrow-schema 56.2.1", - "arrow-select 56.2.1", - "arrow-string 56.2.1", -] - -[[package]] -name = "arrow" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd47f2a6ddc39244bd722a27ee5da66c03369d087b9e024eafdb03e98b98ea7" -dependencies = [ - "arrow-arith 57.3.1", - "arrow-array 57.3.1", - "arrow-buffer 57.3.1", - "arrow-cast 57.3.1", - "arrow-csv 57.3.1", - "arrow-data 57.3.1", - "arrow-ipc 57.3.1", - "arrow-json 57.3.1", - "arrow-ord 57.3.1", - "arrow-row 57.3.1", - "arrow-schema 57.3.1", - "arrow-select 57.3.1", - "arrow-string 57.3.1", -] - [[package]] name = "arrow" version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" dependencies = [ - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-csv 58.3.0", - "arrow-data 58.3.0", - "arrow-ipc 58.3.0", - "arrow-json 58.3.0", - "arrow-ord 58.3.0", - "arrow-row 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", - "arrow-string 58.3.0", -] - -[[package]] -name = "arrow-arith" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce4751cbc4bcccfeeea79df9571ff1dc066d61e44723c7604d11c7937f5b560" -dependencies = [ - "arrow-array 56.2.1", - "arrow-buffer 56.2.1", - "arrow-data 56.2.1", - "arrow-schema 56.2.1", - "chrono", - "num", -] - -[[package]] -name = "arrow-arith" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7bbd679c5418b8639b92be01f361d60013c4906574b578b77b63c78356594c" -dependencies = [ - "arrow-array 57.3.1", - "arrow-buffer 57.3.1", - "arrow-data 57.3.1", - "arrow-schema 57.3.1", - "chrono", - "num-traits", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] @@ -320,48 +250,14 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num-traits", ] -[[package]] -name = "arrow-array" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b02ccba2e977a3aabb4384036109ca32f552399a2bc0588f925f91ed073ce70c" -dependencies = [ - "ahash 0.8.12", - "arrow-buffer 56.2.1", - "arrow-data 56.2.1", - "arrow-schema 56.2.1", - "chrono", - "half", - "hashbrown 0.16.1", - "num", -] - -[[package]] -name = "arrow-array" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8a4ab47b3f3eac60f7fd31b81e9028fda018607bcc63451aca4f2b755269862" -dependencies = [ - "ahash 0.8.12", - "arrow-buffer 57.3.1", - "arrow-data 57.3.1", - "arrow-schema 57.3.1", - "chrono", - "half", - "hashbrown 0.16.1", - "num-complex", - "num-integer", - "num-traits", -] - [[package]] name = "arrow-array" version = "58.3.0" @@ -369,9 +265,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" dependencies = [ "ahash 0.8.12", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", @@ -381,29 +277,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "arrow-buffer" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90f8bece6a9ee316a699fbbfde368a206676a1206ce89b50f07937648e76c3c" -dependencies = [ - "bytes", - "half", - "num", -] - -[[package]] -name = "arrow-buffer" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d18b89b4c4f4811d0858175e79541fe98e33e18db3b011708bc287b1240593f" -dependencies = [ - "bytes", - "half", - "num-bigint", - "num-traits", -] - [[package]] name = "arrow-buffer" version = "58.3.0" @@ -416,61 +289,18 @@ dependencies = [ "num-traits", ] -[[package]] -name = "arrow-cast" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ffe645cfb4e80b1ca37a3a106ce7b4af66ccdd60c655a57e6b9aab096164a7" -dependencies = [ - "arrow-array 56.2.1", - "arrow-buffer 56.2.1", - "arrow-data 56.2.1", - "arrow-schema 56.2.1", - "arrow-select 56.2.1", - "atoi", - "base64", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num", - "ryu", -] - -[[package]] -name = "arrow-cast" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722b5c41dd1d14d0a879a1bce92c6fe33f546101bb2acce57a209825edd075b3" -dependencies = [ - "arrow-array 57.3.1", - "arrow-buffer 57.3.1", - "arrow-data 57.3.1", - "arrow-ord 57.3.1", - "arrow-schema 57.3.1", - "arrow-select 57.3.1", - "atoi", - "base64", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num-traits", - "ryu", -] - [[package]] name = "arrow-cast" version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-ord 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", "atoi", "base64", "chrono", @@ -481,84 +311,29 @@ dependencies = [ "ryu", ] -[[package]] -name = "arrow-csv" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d376e82c15a6298b49a53fbb0d89348db1d5dd3a5147977d62d5516d430cfed3" -dependencies = [ - "arrow-array 56.2.1", - "arrow-cast 56.2.1", - "arrow-schema 56.2.1", - "chrono", - "csv", - "csv-core", - "regex", -] - -[[package]] -name = "arrow-csv" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ddb80a4848e03b1655af496d5ac2563a779e5742fcb48f2ca2e089c9cd2197" -dependencies = [ - "arrow-array 57.3.1", - "arrow-cast 57.3.1", - "arrow-schema 57.3.1", - "chrono", - "csv", - "csv-core", - "regex", -] - [[package]] name = "arrow-csv" version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" dependencies = [ - "arrow-array 58.3.0", - "arrow-cast 58.3.0", - "arrow-schema 58.3.0", + "arrow-array", + "arrow-cast", + "arrow-schema", "chrono", "csv", "csv-core", "regex", ] -[[package]] -name = "arrow-data" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78468c813909465dd0f858950c8a0614eb63608134acf95c602ec21381258b28" -dependencies = [ - "arrow-buffer 56.2.1", - "arrow-schema 56.2.1", - "half", - "num", -] - -[[package]] -name = "arrow-data" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1683705c63dcf0d18972759eda48489028cbbff67af7d6bef2c6b7b74ab778a" -dependencies = [ - "arrow-buffer 57.3.1", - "arrow-schema 57.3.1", - "half", - "num-integer", - "num-traits", -] - [[package]] name = "arrow-data" version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" dependencies = [ - "arrow-buffer 58.3.0", - "arrow-schema 58.3.0", + "arrow-buffer", + "arrow-schema", "half", "num-integer", "num-traits", @@ -570,17 +345,17 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28abfe8bf9f124e5fc83b334af4fa58f8d0323ad25312ccb2d1da50178415704" dependencies = [ - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-data 58.3.0", - "arrow-ipc 58.3.0", - "arrow-ord 58.3.0", - "arrow-row 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", - "arrow-string 58.3.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", "base64", "bytes", "futures", @@ -592,108 +367,34 @@ dependencies = [ "tonic-prost", ] -[[package]] -name = "arrow-ipc" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f88b0fbb33af28089ccd3e4dcd0ff09de46842168d00220b920f7231feddf5" -dependencies = [ - "arrow-array 56.2.1", - "arrow-buffer 56.2.1", - "arrow-data 56.2.1", - "arrow-schema 56.2.1", - "arrow-select 56.2.1", - "flatbuffers", -] - -[[package]] -name = "arrow-ipc" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf72d04c07229fbf4dbebe7145cac37d7cf7ec582fe705c6b92cb314af096ab" -dependencies = [ - "arrow-array 57.3.1", - "arrow-buffer 57.3.1", - "arrow-data 57.3.1", - "arrow-schema 57.3.1", - "arrow-select 57.3.1", - "flatbuffers", -] - [[package]] name = "arrow-ipc" version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "flatbuffers", "lz4_flex 0.13.1", "zstd", ] -[[package]] -name = "arrow-json" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff14ad7669f0742f3c43c606465ad4aad97cfcee24e6317a30f68eba9d75070" -dependencies = [ - "arrow-array 56.2.1", - "arrow-buffer 56.2.1", - "arrow-cast 56.2.1", - "arrow-data 56.2.1", - "arrow-schema 56.2.1", - "chrono", - "half", - "indexmap 2.14.0", - "lexical-core", - "memchr", - "num", - "serde", - "serde_json", - "simdutf8", -] - -[[package]] -name = "arrow-json" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84a905f41fedfcd7679813c89a61dc369c0f932b27aa8dcc6aa051cc781a97d" -dependencies = [ - "arrow-array 57.3.1", - "arrow-buffer 57.3.1", - "arrow-cast 57.3.1", - "arrow-data 57.3.1", - "arrow-schema 57.3.1", - "chrono", - "half", - "indexmap 2.14.0", - "itoa", - "lexical-core", - "memchr", - "num-traits", - "ryu", - "serde_core", - "serde_json", - "simdutf8", -] - [[package]] name = "arrow-json" version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-ord 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", "chrono", "half", "indexmap 2.14.0", @@ -707,69 +408,17 @@ dependencies = [ "simdutf8", ] -[[package]] -name = "arrow-ord" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aed58a38c3db0a2cf75ef70e3cb6bc4bd0da0a3d390de37c36139b31fae826e8" -dependencies = [ - "arrow-array 56.2.1", - "arrow-buffer 56.2.1", - "arrow-data 56.2.1", - "arrow-schema 56.2.1", - "arrow-select 56.2.1", -] - -[[package]] -name = "arrow-ord" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "082342947d4e5a2bcccf029a0a0397e21cb3bb8421edd9571d34fb5dd2670256" -dependencies = [ - "arrow-array 57.3.1", - "arrow-buffer 57.3.1", - "arrow-data 57.3.1", - "arrow-schema 57.3.1", - "arrow-select 57.3.1", -] - [[package]] name = "arrow-ord" version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", -] - -[[package]] -name = "arrow-row" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "079ced0517daf4f09b070d09ff641cee7cc331aa216bebcb25d1a6474ad53086" -dependencies = [ - "arrow-array 56.2.1", - "arrow-buffer 56.2.1", - "arrow-data 56.2.1", - "arrow-schema 56.2.1", - "half", -] - -[[package]] -name = "arrow-row" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a931b520a2a5e22033e01a6f2486b4cdc26f9106b759abeebc320f125e94d7" -dependencies = [ - "arrow-array 57.3.1", - "arrow-buffer 57.3.1", - "arrow-data 57.3.1", - "arrow-schema 57.3.1", - "half", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] @@ -778,25 +427,13 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] -[[package]] -name = "arrow-schema" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a0d5eb3fe25337ff83e8333a08379bdd1540b0961b1c888f6e505d971c198e1" - -[[package]] -name = "arrow-schema" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4cf0d4a6609679e03002167a61074a21d7b1ad9ea65e462b2c0a97f8a3b2bc6" - [[package]] name = "arrow-schema" version = "58.3.0" @@ -809,34 +446,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "arrow-select" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2368a78bd32902dba39d52519d70f63799c8b5dc8a9477129a30c2fd3dc70c19" -dependencies = [ - "ahash 0.8.12", - "arrow-array 56.2.1", - "arrow-buffer 56.2.1", - "arrow-data 56.2.1", - "arrow-schema 56.2.1", - "num", -] - -[[package]] -name = "arrow-select" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b320d86a9806923663bb0fd9baa65ecaba81cb0cd77ff8c1768b9716b4ef891" -dependencies = [ - "ahash 0.8.12", - "arrow-array 57.3.1", - "arrow-buffer 57.3.1", - "arrow-data 57.3.1", - "arrow-schema 57.3.1", - "num-traits", -] - [[package]] name = "arrow-select" version = "58.3.0" @@ -844,58 +453,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" dependencies = [ "ahash 0.8.12", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num-traits", ] -[[package]] -name = "arrow-string" -version = "56.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dece58a130b9187756ded8bc071bd8ee9dd7a146566af244b297c7e632fd1ef7" -dependencies = [ - "arrow-array 56.2.1", - "arrow-buffer 56.2.1", - "arrow-data 56.2.1", - "arrow-schema 56.2.1", - "arrow-select 56.2.1", - "memchr", - "num", - "regex", - "regex-syntax", -] - -[[package]] -name = "arrow-string" -version = "57.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b493e99162e5764077e7823e50ba284858d365922631c7aaefe9487b1abd02c2" -dependencies = [ - "arrow-array 57.3.1", - "arrow-buffer 57.3.1", - "arrow-data 57.3.1", - "arrow-schema 57.3.1", - "arrow-select 57.3.1", - "memchr", - "num-traits", - "regex", - "regex-syntax", -] - [[package]] name = "arrow-string" version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num-traits", "regex", @@ -1061,7 +636,7 @@ dependencies = [ "lz4_flex 0.11.6", "ndarray 0.17.2", "num_cpus", - "object_store 0.13.2", + "object_store", "parking_lot", "rmp-serde", "serde", @@ -1105,17 +680,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "auto_impl" version = "1.3.0" @@ -1217,7 +781,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e3b97b0eb0c163a3a7ed79c496f9b9e6f121259a8434d893c3245b2d9a42c9" dependencies = [ - "arrow 58.3.0", + "arrow", "axum", "bytes", "futures", @@ -1281,9 +845,9 @@ name = "beacon-api" version = "1.8.0" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "arrow-flight", - "arrow-ipc 58.3.0", + "arrow-ipc", "axum", "axum-streams", "base64", @@ -1319,7 +883,7 @@ name = "beacon-arrow-atlas" version = "0.1.0" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "async-trait", "atlas-rust", "beacon-common", @@ -1335,7 +899,7 @@ dependencies = [ "lz4_flex 0.11.6", "moka", "ndarray 0.17.2", - "object_store 0.13.2", + "object_store", "rmp-serde", "serde", "serde_json", @@ -1349,7 +913,7 @@ dependencies = [ name = "beacon-arrow-bbf" version = "1.7.3" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "beacon-binary-format", "beacon-common", @@ -1357,7 +921,7 @@ dependencies = [ "datafusion", "futures", "nd-arrow-array", - "object_store 0.13.2", + "object_store", "parking_lot", "tempfile", "tokio", @@ -1368,13 +932,13 @@ dependencies = [ name = "beacon-arrow-csv" version = "1.7.3" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "beacon-common", "beacon-datafusion-ext", "datafusion", "futures", - "object_store 0.13.2", + "object_store", "tokio", "tracing", ] @@ -1383,7 +947,7 @@ dependencies = [ name = "beacon-arrow-geoparquet" version = "1.7.3" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "beacon-common", "beacon-datafusion-ext", @@ -1394,7 +958,7 @@ dependencies = [ "geoarrow-array", "geoarrow-schema", "geoparquet", - "object_store 0.13.2", + "object_store", "parquet", "serde", "tempfile", @@ -1406,13 +970,13 @@ dependencies = [ name = "beacon-arrow-ipc" version = "1.7.3" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "beacon-common", "beacon-datafusion-ext", "datafusion", "futures", - "object_store 0.13.2", + "object_store", "tokio", "tracing", ] @@ -1422,7 +986,7 @@ name = "beacon-arrow-netcdf" version = "1.7.3" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "async-trait", "beacon-common", "beacon-config", @@ -1441,7 +1005,7 @@ dependencies = [ "netcdf", "netcdf-sys", "num-traits", - "object_store 0.13.2", + "object_store", "ordered-float 5.3.0", "serde", "tempfile", @@ -1454,8 +1018,8 @@ name = "beacon-arrow-odv" version = "1.7.3" dependencies = [ "anyhow", - "arrow 58.3.0", - "arrow-csv 58.3.0", + "arrow", + "arrow-csv", "async-trait", "async_zip", "beacon-common", @@ -1464,7 +1028,7 @@ dependencies = [ "datafusion", "futures", "indexmap 2.14.0", - "object_store 0.13.2", + "object_store", "regex", "serde", "tempfile", @@ -1481,13 +1045,13 @@ dependencies = [ name = "beacon-arrow-parquet" version = "1.7.3" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "beacon-common", "beacon-datafusion-ext", "datafusion", "futures", - "object_store 0.13.2", + "object_store", "tokio", "tracing", ] @@ -1497,7 +1061,7 @@ name = "beacon-arrow-tiff" version = "0.1.0" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "async-tiff", "async-trait", "beacon-common", @@ -1509,7 +1073,7 @@ dependencies = [ "futures", "indexmap 2.14.0", "ndarray 0.17.2", - "object_store 0.13.2", + "object_store", "serde", "tokio", "tracing", @@ -1520,7 +1084,7 @@ name = "beacon-arrow-zarr" version = "1.7.3" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "async-trait", "beacon-common", "beacon-datafusion-ext", @@ -1530,7 +1094,7 @@ dependencies = [ "hifitime", "indexmap 2.14.0", "ndarray 0.17.2", - "object_store 0.13.2", + "object_store", "serde_json", "tempfile", "tokio", @@ -1565,13 +1129,10 @@ name = "beacon-binary-format" version = "2.2.0" dependencies = [ "anyhow", - "arrow 56.2.1", - "arrow 57.3.1", - "arrow 58.3.0", + "arrow", "async-trait", "byteorder", "bytes", - "criterion 0.4.0", "crossbeam", "flume", "futures", @@ -1582,10 +1143,8 @@ dependencies = [ "moka", "munge_macro", "nd-arrow-array", - "object_store 0.12.5", - "object_store 0.13.2", + "object_store", "parking_lot", - "rand 0.9.4", "rkyv 0.8.10", "serde", "serde_json", @@ -1600,7 +1159,7 @@ dependencies = [ name = "beacon-common" version = "1.8.0" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "datafusion", "futures", @@ -1626,7 +1185,7 @@ dependencies = [ "beacon-object-storage", "envconfig", "lazy_static", - "object_store 0.13.2", + "object_store", "thiserror 2.0.18", "tracing", ] @@ -1636,7 +1195,7 @@ name = "beacon-core" version = "1.8.0" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "async-stream", "async-trait", "beacon-arrow-bbf", @@ -1669,7 +1228,7 @@ dependencies = [ "futures", "geodatafusion", "geojson", - "object_store 0.13.2", + "object_store", "parking_lot", "serde", "serde_json", @@ -1688,7 +1247,7 @@ name = "beacon-data-lake" version = "1.7.3" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "async-trait", "beacon-arrow-atlas", "beacon-arrow-bbf", @@ -1709,7 +1268,7 @@ dependencies = [ "beacon-sql-databases", "datafusion", "futures", - "object_store 0.13.2", + "object_store", "parking_lot", "serde", "serde_json", @@ -1725,7 +1284,7 @@ name = "beacon-datafusion-ext" version = "1.7.3" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "arrow-flight", "async-trait", "beacon-common", @@ -1736,7 +1295,7 @@ dependencies = [ "futures", "indexmap 2.14.0", "moka", - "object_store 0.13.2", + "object_store", "ordered-float 5.3.0", "parking_lot", "serde", @@ -1753,7 +1312,7 @@ name = "beacon-delta" version = "0.1.0" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "async-trait", "beacon-common", "beacon-datafusion-ext", @@ -1761,7 +1320,7 @@ dependencies = [ "datafusion", "deltalake", "futures", - "object_store 0.13.2", + "object_store", "serde", "serde_json", "tempfile", @@ -1775,7 +1334,7 @@ name = "beacon-functions" version = "1.7.3" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "beacon-arrow-atlas", "beacon-arrow-bbf", "beacon-arrow-csv", @@ -1799,7 +1358,7 @@ dependencies = [ "gsw", "lazy_static", "lru 0.14.0", - "object_store 0.13.2", + "object_store", "once_cell", "ordered-float 5.3.0", "serde", @@ -1815,7 +1374,7 @@ name = "beacon-iceberg" version = "1.7.3" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "async-trait", "beacon-config", "beacon-datafusion-ext", @@ -1825,7 +1384,7 @@ dependencies = [ "futures", "iceberg-file-catalog", "iceberg-rust", - "object_store 0.13.2", + "object_store", "serde", "serde_json", "tempfile", @@ -1839,7 +1398,7 @@ name = "beacon-lance" version = "0.1.0" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "async-trait", "beacon-datafusion-ext", "datafusion", @@ -1848,7 +1407,7 @@ dependencies = [ "lance-core", "lance-index", "lance-io", - "object_store 0.13.2", + "object_store", "parking_lot", "serde", "serde_json", @@ -1864,8 +1423,8 @@ name = "beacon-mcp" version = "0.1.0" dependencies = [ "anyhow", - "arrow 58.3.0", - "arrow-json 58.3.0", + "arrow", + "arrow-json", "beacon-core", "futures", "http", @@ -1881,11 +1440,11 @@ name = "beacon-nd-array" version = "0.1.0" dependencies = [ "anyhow", - "arrow 58.3.0", - "arrow-ipc 58.3.0", + "arrow", + "arrow-ipc", "async-trait", "bytemuck", - "criterion 0.5.1", + "criterion", "datafusion", "futures", "indexmap 2.14.0", @@ -1904,13 +1463,13 @@ name = "beacon-nd-arrow" version = "0.2.0" dependencies = [ "anyhow", - "arrow 58.3.0", - "arrow-ipc 58.3.0", - "arrow-schema 58.3.0", + "arrow", + "arrow-ipc", + "arrow-schema", "async-trait", "bytemuck", "chrono", - "criterion 0.5.1", + "criterion", "futures", "ndarray 0.17.2", "rand 0.8.6", @@ -1929,7 +1488,7 @@ dependencies = [ "flume", "futures", "notify", - "object_store 0.13.2", + "object_store", "parking_lot", "radix_trie", "smol_str", @@ -1944,7 +1503,7 @@ name = "beacon-sql-databases" version = "0.1.0" dependencies = [ "anyhow", - "arrow 58.3.0", + "arrow", "async-trait", "base64", "beacon-config", @@ -2165,7 +1724,7 @@ version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2235eb320cd7178862a32dd111bd0c0f71a368e393add4914c50129add478eab" dependencies = [ - "arrow 58.3.0", + "arrow", "buoyant_kernel_derive", "bytes", "chrono", @@ -2173,7 +1732,7 @@ dependencies = [ "futures", "indexmap 2.14.0", "itertools 0.14.0", - "object_store 0.13.2", + "object_store", "parquet", "percent-encoding", "rand 0.9.4", @@ -2416,18 +1975,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "clap" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" -dependencies = [ - "bitflags 1.3.2", - "clap_lex 0.2.4", - "indexmap 1.9.3", - "textwrap", -] - [[package]] name = "clap" version = "4.6.1" @@ -2444,16 +1991,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstyle", - "clap_lex 1.1.0", -] - -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", + "clap_lex", ] [[package]] @@ -2664,34 +2202,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" -dependencies = [ - "anes", - "atty", - "cast", - "ciborium", - "clap 3.2.25", - "criterion-plot", - "futures", - "itertools 0.10.5", - "lazy_static", - "num-traits", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "tokio", - "walkdir", -] - [[package]] name = "criterion" version = "0.5.1" @@ -2701,7 +2211,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.6.1", + "clap", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -2967,8 +2477,8 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" dependencies = [ - "arrow 58.3.0", - "arrow-schema 58.3.0", + "arrow", + "arrow-schema", "async-trait", "bytes", "bzip2", @@ -3003,7 +2513,7 @@ dependencies = [ "itertools 0.14.0", "liblzma", "log", - "object_store 0.13.2", + "object_store", "parking_lot", "parquet", "rand 0.9.4", @@ -3022,7 +2532,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "dashmap", "datafusion-common", @@ -3036,7 +2546,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.13.2", + "object_store", "parking_lot", "tokio", ] @@ -3047,7 +2557,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "datafusion-catalog", "datafusion-common", @@ -3061,7 +2571,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.13.2", + "object_store", ] [[package]] @@ -3071,8 +2581,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" dependencies = [ "ahash 0.8.12", - "arrow 58.3.0", - "arrow-ipc 58.3.0", + "arrow", + "arrow-ipc", "chrono", "half", "hashbrown 0.16.1", @@ -3080,7 +2590,7 @@ dependencies = [ "itertools 0.14.0", "libc", "log", - "object_store 0.13.2", + "object_store", "parquet", "paste", "recursive", @@ -3106,7 +2616,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" dependencies = [ - "arrow 58.3.0", + "arrow", "async-compression", "async-trait", "bytes", @@ -3127,7 +2637,7 @@ dependencies = [ "itertools 0.14.0", "liblzma", "log", - "object_store 0.13.2", + "object_store", "rand 0.9.4", "tokio", "tokio-util", @@ -3141,8 +2651,8 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" dependencies = [ - "arrow 58.3.0", - "arrow-ipc 58.3.0", + "arrow", + "arrow-ipc", "async-trait", "bytes", "datafusion-common", @@ -3155,7 +2665,7 @@ dependencies = [ "datafusion-session", "futures", "itertools 0.14.0", - "object_store 0.13.2", + "object_store", "tokio", ] @@ -3165,7 +2675,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "bytes", "datafusion-common", @@ -3177,7 +2687,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store 0.13.2", + "object_store", "regex", "tokio", ] @@ -3188,7 +2698,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "bytes", "datafusion-common", @@ -3200,7 +2710,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store 0.13.2", + "object_store", "serde_json", "tokio", "tokio-stream", @@ -3212,7 +2722,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "bytes", "datafusion-common", @@ -3230,7 +2740,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.13.2", + "object_store", "parking_lot", "parquet", "tokio", @@ -3248,8 +2758,8 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" dependencies = [ - "arrow 58.3.0", - "arrow-buffer 58.3.0", + "arrow", + "arrow-buffer", "async-trait", "chrono", "dashmap", @@ -3258,7 +2768,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "log", - "object_store 0.13.2", + "object_store", "parking_lot", "rand 0.9.4", "tempfile", @@ -3271,7 +2781,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "chrono", "datafusion-common", @@ -3294,7 +2804,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ - "arrow 58.3.0", + "arrow", "datafusion-common", "indexmap 2.14.0", "itertools 0.14.0", @@ -3307,7 +2817,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f88162c5f8650eab86bbe23cac990bdca261c95cf52c05b9a0848ad5b8b2b7f8" dependencies = [ - "arrow-json 58.3.0", + "arrow-json", "async-stream", "async-trait", "datafusion", @@ -3320,8 +2830,8 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" dependencies = [ - "arrow 58.3.0", - "arrow-buffer 58.3.0", + "arrow", + "arrow-buffer", "base64", "blake2", "blake3", @@ -3353,7 +2863,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" dependencies = [ "ahash 0.8.12", - "arrow 58.3.0", + "arrow", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3375,7 +2885,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" dependencies = [ "ahash 0.8.12", - "arrow 58.3.0", + "arrow", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -3387,8 +2897,8 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" dependencies = [ - "arrow 58.3.0", - "arrow-ord 58.3.0", + "arrow", + "arrow-ord", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3412,7 +2922,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "datafusion-catalog", "datafusion-common", @@ -3428,7 +2938,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" dependencies = [ - "arrow 58.3.0", + "arrow", "datafusion-common", "datafusion-doc", "datafusion-expr", @@ -3467,7 +2977,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ - "arrow 58.3.0", + "arrow", "chrono", "datafusion-common", "datafusion-expr", @@ -3488,7 +2998,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" dependencies = [ "ahash 0.8.12", - "arrow 58.3.0", + "arrow", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -3511,7 +3021,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" dependencies = [ - "arrow 58.3.0", + "arrow", "datafusion-common", "datafusion-expr", "datafusion-functions", @@ -3527,7 +3037,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" dependencies = [ "ahash 0.8.12", - "arrow 58.3.0", + "arrow", "chrono", "datafusion-common", "datafusion-expr-common", @@ -3543,7 +3053,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" dependencies = [ - "arrow 58.3.0", + "arrow", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -3563,9 +3073,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" dependencies = [ "ahash 0.8.12", - "arrow 58.3.0", - "arrow-ord 58.3.0", - "arrow-schema 58.3.0", + "arrow", + "arrow-ord", + "arrow-schema", "async-trait", "datafusion-common", "datafusion-common-runtime", @@ -3594,7 +3104,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738" dependencies = [ - "arrow 58.3.0", + "arrow", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -3611,7 +3121,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", - "object_store 0.13.2", + "object_store", "prost", "rand 0.9.4", ] @@ -3622,7 +3132,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9" dependencies = [ - "arrow 58.3.0", + "arrow", "datafusion-common", "prost", ] @@ -3633,7 +3143,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" dependencies = [ - "arrow 58.3.0", + "arrow", "datafusion-common", "datafusion-datasource", "datafusion-expr-common", @@ -3664,7 +3174,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" dependencies = [ - "arrow 58.3.0", + "arrow", "bigdecimal", "chrono", "datafusion-common", @@ -3683,9 +3193,9 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "657cc234eba1c5df39d287a69c58755fa355ff696ec49daee325c0070b7c966a" dependencies = [ - "arrow 58.3.0", - "arrow-json 58.3.0", - "arrow-schema 58.3.0", + "arrow", + "arrow-json", + "arrow-schema", "async-stream", "async-trait", "bb8", @@ -3740,7 +3250,7 @@ dependencies = [ "iceberg-rust", "itertools 0.14.0", "lru 0.16.4", - "object_store 0.13.2", + "object_store", "pin-project-lite", "regex", "serde_json", @@ -3829,17 +3339,17 @@ version = "0.32.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4588e95ff3b2ccdba56d9ec262bd3467c0593000f729402528706f62be8be1ca" dependencies = [ - "arrow 58.3.0", - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-ipc 58.3.0", - "arrow-json 58.3.0", - "arrow-ord 58.3.0", - "arrow-row 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", "async-trait", "buoyant_kernel", "bytes", @@ -3857,7 +3367,7 @@ dependencies = [ "indexmap 2.14.0", "itertools 0.14.0", "num_cpus", - "object_store 0.13.2", + "object_store", "parking_lot", "parquet", "percent-encoding", @@ -4352,7 +3862,7 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcd0ce0249ac12fd44fcde62d435c36d881952c2f0df4d1de24b45e1dbba5ddb" dependencies = [ - "arrow-array 58.3.0", + "arrow-array", "rand 0.9.4", ] @@ -4599,9 +4109,9 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dafe7b7de3fab1a8b7099fd6a6434ca955fa65065f9c19f0f8a133693f3c2b0e" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-schema 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-schema", "geo-traits 0.3.0", "geoarrow-schema", "num-traits", @@ -4615,8 +4125,8 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e4a62ac19c86827c6ec81ea584594b3ee96db5a8119b9774d3466c6b373c434" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", + "arrow-array", + "arrow-buffer", "geo", "geo-traits 0.3.0", "geoarrow-array", @@ -4629,7 +4139,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d4a7edb2a1d87024a93805332a9c8184a0354836271d42c0d18cf628a5e3cd0" dependencies = [ - "arrow-schema 58.3.0", + "arrow-schema", "geo-traits 0.3.0", "serde", "serde_json", @@ -4642,9 +4152,9 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b" dependencies = [ - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-schema 58.3.0", + "arrow-arith", + "arrow-array", + "arrow-schema", "datafusion", "geo", "geo-traits 0.3.0", @@ -4694,11 +4204,11 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a64b758b2b1fc749c5eb212215afa6bc14e1fca93884dac339ab7097ffc20ce1" dependencies = [ - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-ord 58.3.0", - "arrow-schema 58.3.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "futures", "geo-traits 0.3.0", "geo-types", @@ -4952,15 +4462,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.5.2" @@ -5274,7 +4775,7 @@ dependencies = [ "async-trait", "futures", "iceberg-rust", - "object_store 0.13.2", + "object_store", "serde_json", "thiserror 2.0.18", "url", @@ -5288,7 +4789,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f28513696595ea087a33a8a9cb0d795c912569c398a08226a8fcc9bb12d2b898" dependencies = [ "apache-avro", - "arrow 58.3.0", + "arrow", "async-trait", "bytes", "derive-getters", @@ -5301,7 +4802,7 @@ dependencies = [ "lazy_static", "lru 0.16.4", "murmur3", - "object_store 0.13.2", + "object_store", "parquet", "pin-project-lite", "regex", @@ -5325,7 +4826,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72d1777b6a77e205857d034da0e07c9de391e126a2caaa343adbe89471b72de9" dependencies = [ "apache-avro", - "arrow-schema 58.3.0", + "arrow-schema", "chrono", "derive-getters", "derive_builder", @@ -5591,7 +5092,7 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ - "hermit-abi 0.5.2", + "hermit-abi", "libc", "windows-sys 0.61.2", ] @@ -5837,16 +5338,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3944aca86f4c78f4da04af1c2bf33e664a2826b7af72972ad200d6b9de59019f" dependencies = [ "arc-swap", - "arrow 58.3.0", - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-ipc 58.3.0", - "arrow-ord 58.3.0", - "arrow-row 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", "async-recursion", "async-trait", "async_cell", @@ -5881,7 +5382,7 @@ dependencies = [ "lance-tokenizer", "log", "moka", - "object_store 0.13.2", + "object_store", "permutation", "pin-project", "prost", @@ -5908,13 +5409,13 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "253f4a0a70580c985b91e65e9ca6cad644825a4078de28d8efbacf3ffbd7ecdc" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-ipc 58.3.0", - "arrow-ord 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "arrow-select", "bytes", "futures", "getrandom 0.2.17", @@ -5941,9 +5442,9 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13f84020da5a484e2f07dd1796e09785ed7cd889857ebc4cb77e32ef214ee594" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-schema 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-schema", "async-trait", "byteorder", "bytes", @@ -5957,7 +5458,7 @@ dependencies = [ "log", "moka", "num_cpus", - "object_store 0.13.2", + "object_store", "pin-project", "prost", "rand 0.9.4", @@ -5978,13 +5479,13 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7460597a66534a75987993d4dac5bc330586d99c5b79ae73367dbcbd4e29e576" dependencies = [ - "arrow 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-ord 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", "async-trait", "chrono", "datafusion", @@ -6010,10 +5511,10 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046f5506ed2271cd941a050de7bf535dd3aedc291aadec836a63fa56c5926e3b" dependencies = [ - "arrow 58.3.0", - "arrow-array 58.3.0", - "arrow-cast 58.3.0", - "arrow-schema 58.3.0", + "arrow", + "arrow-array", + "arrow-cast", + "arrow-schema", "chrono", "futures", "half", @@ -6030,13 +5531,13 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7af54edf43dcf9d6a56cc636eb35d457e68373c6448dca3f0891b3325b4a24e6" dependencies = [ - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", "bytemuck", "byteorder", "bytes", @@ -6067,12 +5568,12 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0772ae2d6207995dc1eb28aff9507f78e90b3362b58f311da001e9dc25f3d736" dependencies = [ - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "async-recursion", "async-trait", "byteorder", @@ -6086,7 +5587,7 @@ dependencies = [ "lance-io", "log", "num-traits", - "object_store 0.13.2", + "object_store", "prost", "prost-build", "prost-types", @@ -6101,12 +5602,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e71fbfb51096a903cb524fe0da716f5f15fbc4a6b6f84cd6dec21abf319c5e84" dependencies = [ "arc-swap", - "arrow 58.3.0", - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-ord 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow", + "arrow-arith", + "arrow-array", + "arrow-ord", + "arrow-schema", + "arrow-select", "async-channel", "async-recursion", "async-trait", @@ -6140,7 +5641,7 @@ dependencies = [ "log", "ndarray 0.16.1", "num-traits", - "object_store 0.13.2", + "object_store", "prost", "prost-build", "prost-types", @@ -6165,14 +5666,14 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab8c98ef1b870b20541d27f3ca4efdf7c9f5c25214233be07d231ba88900219" dependencies = [ - "arrow 58.3.0", - "arrow-arith 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-cast 58.3.0", - "arrow-data 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", "async-recursion", "async-trait", "byteorder", @@ -6187,7 +5688,7 @@ dependencies = [ "lance-namespace", "log", "moka", - "object_store 0.13.2", + "object_store", "path_abs", "pin-project", "prost", @@ -6205,9 +5706,9 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b4c51cad0ac780b02dc4da48528479e7693c03e8d05390510bbc69ca2a9a1f1" dependencies = [ - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-schema 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-schema", "cc", "deepsize", "half", @@ -6223,7 +5724,7 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "014e8332ca0615506342e0d3af608639864b68396973be14239f09c9f21f1fc2" dependencies = [ - "arrow 58.3.0", + "arrow", "async-trait", "bytes", "lance-core", @@ -6251,11 +5752,11 @@ version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b16f1355904aea4ebb04ffc70c58c97901e10bde44452b4b021de4a1f329250d" dependencies = [ - "arrow 58.3.0", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-ipc 58.3.0", - "arrow-schema 58.3.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ipc", + "arrow-schema", "async-trait", "byteorder", "bytes", @@ -6267,7 +5768,7 @@ dependencies = [ "lance-file", "lance-io", "log", - "object_store 0.13.2", + "object_store", "prost", "prost-build", "prost-types", @@ -6903,9 +6404,7 @@ name = "nd-arrow-array" version = "2.0.0" source = "git+https://github.com/maris-development/nd-arrow-array.git?branch=main#6693aa470b499f1baa22f324610780dae6196a9e" dependencies = [ - "arrow 56.2.1", - "arrow 57.3.1", - "arrow 58.3.0", + "arrow", "thiserror 2.0.18", ] @@ -7135,7 +6634,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi 0.5.2", + "hermit-abi", "libc", ] @@ -7198,42 +6697,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "object_store" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" -dependencies = [ - "async-trait", - "base64", - "bytes", - "chrono", - "form_urlencoded", - "futures", - "http", - "http-body-util", - "humantime", - "hyper", - "itertools 0.14.0", - "md-5 0.10.6", - "parking_lot", - "percent-encoding", - "quick-xml 0.38.4", - "rand 0.9.4", - "reqwest 0.12.28", - "ring", - "serde", - "serde_json", - "serde_urlencoded", - "thiserror 2.0.18", - "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", -] - [[package]] name = "object_store" version = "0.13.2" @@ -7365,12 +6828,6 @@ dependencies = [ "serde", ] -[[package]] -name = "os_str_bytes" -version = "6.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" - [[package]] name = "page_size" version = "0.6.0" @@ -7417,12 +6874,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" dependencies = [ "ahash 0.8.12", - "arrow-array 58.3.0", - "arrow-buffer 58.3.0", - "arrow-data 58.3.0", - "arrow-ipc 58.3.0", - "arrow-schema 58.3.0", - "arrow-select 58.3.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", "base64", "brotli", "bytes", @@ -7435,7 +6892,7 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "object_store 0.13.2", + "object_store", "paste", "seq-macro", "simdutf8", @@ -7602,34 +7059,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - [[package]] name = "poly1305" version = "0.8.0" @@ -7936,16 +7365,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "quick-xml" -version = "0.38.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.39.4" @@ -9594,12 +9013,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "textwrap" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" - [[package]] name = "thiserror" version = "1.0.69" @@ -11403,7 +10816,7 @@ checksum = "ba1662bcdc585be1923a8b98ee5d6314d19af8c5775d6516a69ca33fd88b46af" dependencies = [ "async-trait", "futures", - "object_store 0.13.2", + "object_store", "zarrs_storage", ] diff --git a/Cargo.toml b/Cargo.toml index 06dadfa8..66a6c498 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,12 @@ # Edition-2024 members imply resolver 3; set it explicitly so features aren't # over-unified across normal/build/dev deps (fewer features compiled). resolver = "3" -members = ["beacon-api", "beacon-mcp", "beacon-auth", "beacon-file-formats/beacon-arrow-netcdf", "beacon-file-formats/beacon-arrow-odv", "beacon-common", "beacon-config", "beacon-core", "beacon-functions", "beacon-data-lake", "beacon-sql-databases", "beacon-file-formats/beacon-delta", "beacon-file-formats/beacon-arrow-zarr", "beacon-file-formats/beacon-binary-format", "beacon-object-storage", "beacon-file-formats/beacon-nd-arrow", "beacon-datafusion-ext", "beacon-file-formats/beacon-iceberg", "beacon-file-formats/beacon-lance", "beacon-file-formats/beacon-nd-array", "beacon-file-formats/beacon-arrow-tiff", "beacon-file-formats/beacon-arrow-atlas", "beacon-file-formats/beacon-arrow-geoparquet", "beacon-file-formats/beacon-arrow-bbf", "beacon-file-formats/beacon-arrow-ipc", "beacon-file-formats/beacon-arrow-csv", "beacon-file-formats/beacon-arrow-parquet"] -exclude = ["beacon-file-formats/beacon-binary-format-toolbox"] +members = ["beacon-api", "beacon-mcp", "beacon-auth", "beacon-file-formats/beacon-arrow-netcdf", "beacon-file-formats/beacon-arrow-odv", "beacon-common", "beacon-config", "beacon-core", "beacon-functions", "beacon-data-lake", "beacon-sql-databases", "beacon-file-formats/beacon-delta", "beacon-file-formats/beacon-arrow-zarr", "beacon-object-storage", "beacon-file-formats/beacon-nd-arrow", "beacon-datafusion-ext", "beacon-file-formats/beacon-iceberg", "beacon-file-formats/beacon-lance", "beacon-file-formats/beacon-nd-array", "beacon-file-formats/beacon-arrow-tiff", "beacon-file-formats/beacon-arrow-atlas", "beacon-file-formats/beacon-arrow-geoparquet", "beacon-file-formats/beacon-arrow-bbf", "beacon-file-formats/beacon-arrow-ipc", "beacon-file-formats/beacon-arrow-csv", "beacon-file-formats/beacon-arrow-parquet"] +# beacon-binary-format is a git submodule vendored from its own upstream repo. +# It is excluded from the workspace so workspace-wide tooling (notably +# `cargo fmt --all`) does not touch code we don't own here; it is still built +# as a path dependency of beacon-arrow-bbf, and formatted in its own repo. +exclude = ["beacon-file-formats/beacon-binary-format-toolbox", "beacon-file-formats/beacon-binary-format"] [workspace.dependencies] tokio = {version = "1.47.1", features = ["full"]} diff --git a/Makefile b/Makefile index 34a69ac4..90df5487 100644 --- a/Makefile +++ b/Makefile @@ -18,12 +18,19 @@ WEB_DIR ?= clients/beacon-web/dist export BEACON_ADMIN_USERNAME export BEACON_ADMIN_PASSWORD -.PHONY: help ui-deps ui run serve dev-api dev-ui clean-ui +.PHONY: help hooks fmt ui-deps ui run serve dev-api dev-ui clean-ui help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-10s\033[0m %s\n", $$1, $$2}' +hooks: ## Install the git pre-commit hook (rustfmt check on staged Rust) + git config core.hooksPath .githooks + @echo "Installed .githooks (core.hooksPath). Commits now run the rustfmt check." + +fmt: ## Format all workspace crates with rustfmt (skips the vendored submodule) + cargo fmt + ui-deps: ## Install JS workspace dependencies cd clients && npm install diff --git a/README.md b/README.md index 7cc06673..e1ef9ee3 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,8 @@ cargo test Issues and pull requests are welcome on [GitHub](https://github.com/maris-development/beacon/issues). For larger changes, please open an issue first to discuss the approach. +See [CONTRIBUTING.md](CONTRIBUTING.md) for prerequisites (toolchain, submodules, system libraries), the formatting/lint/test checks CI enforces, and how to install the pre-commit hook. + ## License Beacon is licensed under the **GNU Affero General Public License v3.0** (AGPL-3.0). See [LICENSE](LICENSE) for the full text. diff --git a/beacon-api/src/axum/admin/datasets.rs b/beacon-api/src/axum/admin/datasets.rs index e03bc1c1..c3c49d5c 100644 --- a/beacon-api/src/axum/admin/datasets.rs +++ b/beacon-api/src/axum/admin/datasets.rs @@ -94,7 +94,10 @@ pub(crate) async fn download_dataset( State(state): State>, Query(params): Query, ) -> Result { - let result = state.download_dataset(¶ms.path).await.map_err(file_error)?; + let result = state + .download_dataset(¶ms.path) + .await + .map_err(file_error)?; let size = result.meta.size; let filename = attachment_filename(¶ms.path); let body = Body::from_stream(result.into_stream()); @@ -230,7 +233,10 @@ pub(crate) async fn complete_upload( Query(params): Query, ) -> Result, (StatusCode, String)> { let id = parse_upload_id(¶ms.upload_id)?; - let result = state.complete_dataset_upload(id).await.map_err(file_error)?; + let result = state + .complete_dataset_upload(id) + .await + .map_err(file_error)?; Ok(Json(result)) } @@ -281,7 +287,10 @@ pub(crate) async fn delete_dataset( State(state): State>, Query(params): Query, ) -> Result { - state.delete_dataset(¶ms.path).await.map_err(file_error)?; + state + .delete_dataset(¶ms.path) + .await + .map_err(file_error)?; Ok(StatusCode::NO_CONTENT) } diff --git a/beacon-api/src/axum/admin/mod.rs b/beacon-api/src/axum/admin/mod.rs index dc9f9d97..901c7ec9 100644 --- a/beacon-api/src/axum/admin/mod.rs +++ b/beacon-api/src/axum/admin/mod.rs @@ -36,10 +36,7 @@ pub(crate) fn setup_admin_router() -> (Router>, utoipa::openapi::Op let (admin_router, admin_api) = OpenApiRouter::with_openapi(AdminApiDoc::openapi()) .routes(routes!(check::check)) .routes(routes!(crawlers::create_crawler, crawlers::list_crawlers)) - .routes(routes!( - crawlers::get_crawler, - crawlers::drop_crawler - )) + .routes(routes!(crawlers::get_crawler, crawlers::drop_crawler)) .routes(routes!(crawlers::run_crawler)) .routes(routes!(external_tables::create_external_table)) .routes(routes!(datasets::upload_dataset)) diff --git a/beacon-api/src/axum/admin_datasets_http_tests.rs b/beacon-api/src/axum/admin_datasets_http_tests.rs index 17d8040d..74a545ed 100644 --- a/beacon-api/src/axum/admin_datasets_http_tests.rs +++ b/beacon-api/src/axum/admin_datasets_http_tests.rs @@ -259,9 +259,7 @@ async fn chunked_upload_round_trip_and_download() { &router, request( "PUT", - &format!( - "/api/admin/datasets/upload/part?upload_id={upload_id}&part_number={n}" - ), + &format!("/api/admin/datasets/upload/part?upload_id={upload_id}&part_number={n}"), Some(&admin), Body::from(payload), ), diff --git a/beacon-api/src/axum/auth_http_tests.rs b/beacon-api/src/axum/auth_http_tests.rs index d3cb7534..fadf0792 100644 --- a/beacon-api/src/axum/auth_http_tests.rs +++ b/beacon-api/src/axum/auth_http_tests.rs @@ -102,7 +102,11 @@ async fn admin_route_requires_a_super_user() { ); // Valid but non-super credentials → 403. assert_eq!( - status(&router, get("/api/admin/check", Some(&basic("alice", "pw")))).await, + status( + &router, + get("/api/admin/check", Some(&basic("alice", "pw"))) + ) + .await, StatusCode::FORBIDDEN ); // Wrong password → 401. @@ -180,19 +184,31 @@ async fn enforced_http_query_respects_table_grants() { seed(&runtime, "CREATE ROLE reader").await; seed(&runtime, "CREATE USER alice WITH PASSWORD 'pw'").await; seed(&runtime, "GRANT ROLE reader TO USER alice").await; - seed(&runtime, &format!("GRANT SELECT ON TABLE {t1} TO ROLE reader")).await; + seed( + &runtime, + &format!("GRANT SELECT ON TABLE {t1} TO ROLE reader"), + ) + .await; let router = setup_router(runtime, config).unwrap(); let auth = basic("alice", "pw"); // Granted table → 200. assert_eq!( - status(&router, post_query(&format!("SELECT * FROM {t1}"), Some(&auth))).await, + status( + &router, + post_query(&format!("SELECT * FROM {t1}"), Some(&auth)) + ) + .await, StatusCode::OK ); // Ungranted table → permission denied, surfaced as 400. assert_eq!( - status(&router, post_query(&format!("SELECT * FROM {t2}"), Some(&auth))).await, + status( + &router, + post_query(&format!("SELECT * FROM {t2}"), Some(&auth)) + ) + .await, StatusCode::BAD_REQUEST ); } diff --git a/beacon-api/src/axum/client/datasets.rs b/beacon-api/src/axum/client/datasets.rs index 2f722bcf..3d0aa837 100644 --- a/beacon-api/src/axum/client/datasets.rs +++ b/beacon-api/src/axum/client/datasets.rs @@ -29,8 +29,8 @@ pub struct ListDatasetsQuery { #[tracing::instrument(level = "info", skip(state))] #[utoipa::path( tag = "datasets", - get, - path = "/api/datasets", + get, + path = "/api/datasets", params(ListDatasetsQuery), responses( (status = 200, description = "List of dataset file paths", body = Vec), @@ -111,7 +111,7 @@ pub struct ListDatasetSchemaQuery { #[tracing::instrument(level = "info", skip(state))] #[utoipa::path( tag = "datasets", - get, + get, path = "/api/dataset-schema", params(ListDatasetSchemaQuery), responses( @@ -173,4 +173,4 @@ pub(crate) async fn total_datasets( )) } } -} \ No newline at end of file +} diff --git a/beacon-api/src/axum/client/functions.rs b/beacon-api/src/axum/client/functions.rs index 91b6b2be..9e1e9c7a 100644 --- a/beacon-api/src/axum/client/functions.rs +++ b/beacon-api/src/axum/client/functions.rs @@ -10,7 +10,7 @@ use beacon_core::runtime::Runtime; #[tracing::instrument(level = "info", skip(state))] #[utoipa::path( tag = "functions", - get, + get, path = "/api/functions", responses((status = 200, description = "Available scalar/aggregate functions with documentation", body = Vec)), security( @@ -42,4 +42,4 @@ pub(crate) async fn list_table_functions( ) -> Json> { let functions = state.list_table_functions(); Json(functions) -} \ No newline at end of file +} diff --git a/beacon-api/src/axum/client/info.rs b/beacon-api/src/axum/client/info.rs index 47ec931d..779505f2 100644 --- a/beacon-api/src/axum/client/info.rs +++ b/beacon-api/src/axum/client/info.rs @@ -9,7 +9,7 @@ use beacon_core::{runtime::Runtime, sys::SystemInfo}; #[tracing::instrument(level = "info", skip(state))] #[utoipa::path( tag = "system", - get, + get, path = "/api/info", responses((status = 200, description = "Beacon runtime system information", body = SystemInfo)), security( @@ -21,4 +21,4 @@ use beacon_core::{runtime::Runtime, sys::SystemInfo}; pub(crate) async fn system_info(State(state): State>) -> Json { let info = state.system_info(); Json(info) -} \ No newline at end of file +} diff --git a/beacon-api/src/axum/client/mod.rs b/beacon-api/src/axum/client/mod.rs index b6e74535..24fcfd1a 100644 --- a/beacon-api/src/axum/client/mod.rs +++ b/beacon-api/src/axum/client/mod.rs @@ -83,9 +83,8 @@ mod tests { // The /api/query request body references the real Query schema rather // than an opaque object. - let request_ref = spec.pointer( - "/paths/~1api~1query/post/requestBody/content/application~1json/schema/$ref", - ); + let request_ref = spec + .pointer("/paths/~1api~1query/post/requestBody/content/application~1json/schema/$ref"); assert_eq!( request_ref.and_then(|r| r.as_str()), Some("#/components/schemas/Query"), diff --git a/beacon-api/src/axum/client/query.rs b/beacon-api/src/axum/client/query.rs index 401398d0..19e5b734 100644 --- a/beacon-api/src/axum/client/query.rs +++ b/beacon-api/src/axum/client/query.rs @@ -59,9 +59,7 @@ pub(crate) async fn query( // SQL over the HTTP client API is gated by `sql.enable` (JSON is always // allowed); the Flight SQL transport has its own `flight_sql.enable`. - if matches!(query.inner, beacon_core::query::InnerQuery::Sql(_)) - && !state.config().sql.enable - { + if matches!(query.inner, beacon_core::query::InnerQuery::Sql(_)) && !state.config().sql.enable { return Err(( StatusCode::BAD_REQUEST, Json("SQL queries are not enabled".to_string()), @@ -346,9 +344,7 @@ pub(crate) async fn explain_analyze_query( // EXPLAIN ANALYZE executes the query, so it is gated by `sql.enable` exactly // like `/api/query` (JSON is always allowed). Without this, SQL could be run // through this endpoint while SQL is disabled, bypassing the restriction. - if matches!(query.inner, beacon_core::query::InnerQuery::Sql(_)) - && !state.config().sql.enable - { + if matches!(query.inner, beacon_core::query::InnerQuery::Sql(_)) && !state.config().sql.enable { return Err(( StatusCode::BAD_REQUEST, Json("SQL queries are not enabled".to_string()), diff --git a/beacon-api/src/axum/client/tables.rs b/beacon-api/src/axum/client/tables.rs index ad972a79..abf96907 100644 --- a/beacon-api/src/axum/client/tables.rs +++ b/beacon-api/src/axum/client/tables.rs @@ -15,7 +15,7 @@ use utoipa::{IntoParams, ToSchema}; #[tracing::instrument(level = "info", skip(state))] #[utoipa::path( tag = "tables", - get, + get, path = "/api/tables", responses((status = 200, description = "List of registered table names", body = Vec)), security( @@ -42,7 +42,7 @@ pub(crate) struct TableWithSchema { #[tracing::instrument(level = "info", skip(state))] #[utoipa::path( tag = "tables", - get, + get, path = "/api/tables-with-schema", responses((status = 200, description = "Registered tables with their Arrow schemas", body = Vec)), security( @@ -79,7 +79,7 @@ pub struct ListTableSchemaQuery { #[tracing::instrument(level = "info", skip(state))] #[utoipa::path( tag = "tables", - get, + get, path = "/api/table-schema", params(ListTableSchemaQuery), responses( @@ -165,9 +165,7 @@ pub(crate) async fn list_table_extensions( ("bearer" = []) ) )] -pub(crate) async fn default_table_schema( - State(state): State>, -) -> Json { +pub(crate) async fn default_table_schema(State(state): State>) -> Json { let result = state.list_default_table_schema_view().await; Json(result) } @@ -188,4 +186,4 @@ pub(crate) async fn default_table_schema( pub(crate) async fn default_table(State(state): State>) -> Json { let result = state.default_table(); Json(result) -} \ No newline at end of file +} diff --git a/beacon-api/src/axum/mod.rs b/beacon-api/src/axum/mod.rs index d349a41b..af2d23ed 100644 --- a/beacon-api/src/axum/mod.rs +++ b/beacon-api/src/axum/mod.rs @@ -11,4 +11,4 @@ mod client; mod rbac_http_tests; mod router; -pub(crate) use router::setup_router; \ No newline at end of file +pub(crate) use router::setup_router; diff --git a/beacon-api/src/axum/rbac_http_tests.rs b/beacon-api/src/axum/rbac_http_tests.rs index 2e4504c9..60424e03 100644 --- a/beacon-api/src/axum/rbac_http_tests.rs +++ b/beacon-api/src/axum/rbac_http_tests.rs @@ -123,7 +123,9 @@ async fn auth_endpoints_gated_and_list_default_principals() { StatusCode::UNAUTHORIZED ); assert_eq!( - send(&router, get("/api/admin/auth/users", Some(&alice))).await.0, + send(&router, get("/api/admin/auth/users", Some(&alice))) + .await + .0, StatusCode::FORBIDDEN ); let (status, body) = send(&router, get("/api/admin/auth/users", Some(&admin))).await; @@ -156,14 +158,33 @@ async fn rbac_lifecycle_reflected_in_endpoints() { // Create + grant + deny + assign. admin_ok(&router, &admin, "CREATE ROLE reader").await; - admin_ok(&router, &admin, "GRANT SELECT ON TABLE observations TO ROLE reader").await; - admin_ok(&router, &admin, "GRANT SELECT ON PATH 'argo/**/*.nc' TO ROLE reader").await; - admin_ok(&router, &admin, "DENY SELECT ON TABLE secret TO ROLE reader").await; + admin_ok( + &router, + &admin, + "GRANT SELECT ON TABLE observations TO ROLE reader", + ) + .await; + admin_ok( + &router, + &admin, + "GRANT SELECT ON PATH 'argo/**/*.nc' TO ROLE reader", + ) + .await; + admin_ok( + &router, + &admin, + "DENY SELECT ON TABLE secret TO ROLE reader", + ) + .await; admin_ok(&router, &admin, "CREATE USER alice WITH PASSWORD 'pw'").await; admin_ok(&router, &admin, "GRANT ROLE reader TO USER alice").await; // Roles endpoint reflects the grants and denies. - let roles = json(&send(&router, get("/api/admin/auth/roles", Some(&admin))).await.1); + let roles = json( + &send(&router, get("/api/admin/auth/roles", Some(&admin))) + .await + .1, + ); let reader = roles .as_array() .unwrap() @@ -172,7 +193,9 @@ async fn rbac_lifecycle_reflected_in_endpoints() { .expect("reader role present"); let grants = reader["grants"].as_array().unwrap(); assert!(grants.iter().any(|g| { - g["privilege"] == "SELECT" && g["target_type"] == "table" && g["target_value"] == "observations" + g["privilege"] == "SELECT" + && g["target_type"] == "table" + && g["target_value"] == "observations" })); assert!(grants .iter() @@ -183,7 +206,11 @@ async fn rbac_lifecycle_reflected_in_endpoints() { .any(|d| d["target_type"] == "table" && d["target_value"] == "secret")); // Users endpoint reflects the role assignment. - let users = json(&send(&router, get("/api/admin/auth/users", Some(&admin))).await.1); + let users = json( + &send(&router, get("/api/admin/auth/users", Some(&admin))) + .await + .1, + ); let alice = users .as_array() .unwrap() @@ -193,15 +220,37 @@ async fn rbac_lifecycle_reflected_in_endpoints() { assert_eq!(alice["roles"], serde_json::json!(["reader"])); // Revoke + drop, then confirm the state is gone. - admin_ok(&router, &admin, "REVOKE SELECT ON TABLE observations FROM ROLE reader").await; - admin_ok(&router, &admin, "REVOKE DENY SELECT ON TABLE secret FROM ROLE reader").await; + admin_ok( + &router, + &admin, + "REVOKE SELECT ON TABLE observations FROM ROLE reader", + ) + .await; + admin_ok( + &router, + &admin, + "REVOKE DENY SELECT ON TABLE secret FROM ROLE reader", + ) + .await; admin_ok(&router, &admin, "REVOKE ROLE reader FROM USER alice").await; admin_ok(&router, &admin, "DROP USER alice").await; - let users = json(&send(&router, get("/api/admin/auth/users", Some(&admin))).await.1); - assert!(!users.as_array().unwrap().iter().any(|u| u["username"] == "alice")); + let users = json( + &send(&router, get("/api/admin/auth/users", Some(&admin))) + .await + .1, + ); + assert!(!users + .as_array() + .unwrap() + .iter() + .any(|u| u["username"] == "alice")); - let roles = json(&send(&router, get("/api/admin/auth/roles", Some(&admin))).await.1); + let roles = json( + &send(&router, get("/api/admin/auth/roles", Some(&admin))) + .await + .1, + ); let reader = roles .as_array() .unwrap() @@ -216,8 +265,16 @@ async fn rbac_lifecycle_reflected_in_endpoints() { assert!(reader["denies"].as_array().unwrap().is_empty()); admin_ok(&router, &admin, "DROP ROLE reader").await; - let roles = json(&send(&router, get("/api/admin/auth/roles", Some(&admin))).await.1); - assert!(!roles.as_array().unwrap().iter().any(|r| r["name"] == "reader")); + let roles = json( + &send(&router, get("/api/admin/auth/roles", Some(&admin))) + .await + .1, + ); + assert!(!roles + .as_array() + .unwrap() + .iter() + .any(|r| r["name"] == "reader")); } #[tokio::test(flavor = "multi_thread")] @@ -231,18 +288,30 @@ async fn model_guards_are_enforced() { admin_ok(&router, &admin, "CREATE ROLE reader").await; // Roles are read-only: only SELECT may be granted. - let bad = send(&router, post_query("GRANT INSERT ON TABLE t TO ROLE reader", Some(&admin))).await; + let bad = send( + &router, + post_query("GRANT INSERT ON TABLE t TO ROLE reader", Some(&admin)), + ) + .await; assert_eq!(bad.0, StatusCode::BAD_REQUEST); // The super-user username is reserved (can't be created or dropped via SQL). assert_eq!( - send(&router, post_query(&format!("DROP USER {admin_name}"), Some(&admin))).await.0, + send( + &router, + post_query(&format!("DROP USER {admin_name}"), Some(&admin)) + ) + .await + .0, StatusCode::BAD_REQUEST ); assert_eq!( send( &router, - post_query(&format!("CREATE USER {admin_name} WITH PASSWORD 'x'"), Some(&admin)) + post_query( + &format!("CREATE USER {admin_name} WITH PASSWORD 'x'"), + Some(&admin) + ) ) .await .0, @@ -251,11 +320,21 @@ async fn model_guards_are_enforced() { // The anonymous user can't be deleted while anonymous access is enabled. assert_eq!( - send(&router, post_query("DROP USER anonymous", Some(&admin))).await.0, + send(&router, post_query("DROP USER anonymous", Some(&admin))) + .await + .0, StatusCode::BAD_REQUEST ); - let users = json(&send(&router, get("/api/admin/auth/users", Some(&admin))).await.1); - assert!(users.as_array().unwrap().iter().any(|u| u["username"] == "anonymous")); + let users = json( + &send(&router, get("/api/admin/auth/users", Some(&admin))) + .await + .1, + ); + assert!(users + .as_array() + .unwrap() + .iter() + .any(|u| u["username"] == "anonymous")); } #[tokio::test(flavor = "multi_thread")] @@ -270,12 +349,16 @@ async fn non_super_user_cannot_manage_or_enumerate_auth() { // Auth DDL requires the super-user → 400 for bob. assert_eq!( - send(&router, post_query("CREATE ROLE hacker", Some(&bob))).await.0, + send(&router, post_query("CREATE ROLE hacker", Some(&bob))) + .await + .0, StatusCode::BAD_REQUEST ); // The admin enumeration endpoints reject non-super principals → 403. assert_eq!( - send(&router, get("/api/admin/auth/roles", Some(&bob))).await.0, + send(&router, get("/api/admin/auth/roles", Some(&bob))) + .await + .0, StatusCode::FORBIDDEN ); } @@ -305,22 +388,47 @@ async fn enforcement_grants_allow_and_denies_block() { // The global grant lets alice read both tables. assert_eq!( - send(&router, post_query(&format!("SELECT * FROM {t1}"), Some(&alice))).await.0, + send( + &router, + post_query(&format!("SELECT * FROM {t1}"), Some(&alice)) + ) + .await + .0, StatusCode::OK ); assert_eq!( - send(&router, post_query(&format!("SELECT * FROM {t2}"), Some(&alice))).await.0, + send( + &router, + post_query(&format!("SELECT * FROM {t2}"), Some(&alice)) + ) + .await + .0, StatusCode::OK ); // A deny on t1 wins over the grant; t2 is still readable. - admin_ok(&router, &admin, &format!("DENY SELECT ON TABLE {t1} TO ROLE reader")).await; + admin_ok( + &router, + &admin, + &format!("DENY SELECT ON TABLE {t1} TO ROLE reader"), + ) + .await; assert_eq!( - send(&router, post_query(&format!("SELECT * FROM {t1}"), Some(&alice))).await.0, + send( + &router, + post_query(&format!("SELECT * FROM {t1}"), Some(&alice)) + ) + .await + .0, StatusCode::BAD_REQUEST ); assert_eq!( - send(&router, post_query(&format!("SELECT * FROM {t2}"), Some(&alice))).await.0, + send( + &router, + post_query(&format!("SELECT * FROM {t2}"), Some(&alice)) + ) + .await + .0, StatusCode::OK ); @@ -328,7 +436,12 @@ async fn enforcement_grants_allow_and_denies_block() { admin_ok(&router, &admin, "CREATE USER mallory WITH PASSWORD 'pw'").await; let mallory = basic("mallory", "pw"); assert_eq!( - send(&router, post_query(&format!("SELECT * FROM {t2}"), Some(&mallory))).await.0, + send( + &router, + post_query(&format!("SELECT * FROM {t2}"), Some(&mallory)) + ) + .await + .0, StatusCode::BAD_REQUEST ); } diff --git a/beacon-api/src/axum/router.rs b/beacon-api/src/axum/router.rs index 901755b0..9e62be3d 100644 --- a/beacon-api/src/axum/router.rs +++ b/beacon-api/src/axum/router.rs @@ -77,7 +77,12 @@ pub(crate) fn setup_router( // calls execute under the caller's identity (or the anonymous principal when // enabled) and per-user RBAC applies at query time. let mcp_enabled = std::env::var("BEACON_MCP_ENABLED") - .map(|v| !matches!(v.trim().to_ascii_lowercase().as_str(), "false" | "0" | "off")) + .map(|v| { + !matches!( + v.trim().to_ascii_lowercase().as_str(), + "false" | "0" | "off" + ) + }) .unwrap_or(true); let mut router = client_router.merge(admin_router); if mcp_enabled { diff --git a/beacon-api/src/flight_sql/auth.rs b/beacon-api/src/flight_sql/auth.rs index 71a9458d..91b77f1b 100644 --- a/beacon-api/src/flight_sql/auth.rs +++ b/beacon-api/src/flight_sql/auth.rs @@ -53,11 +53,7 @@ pub(super) struct Authenticator { impl Authenticator { /// Creates a new authenticator with the given anonymous-access policy and token TTL, resolving /// credentials against the shared runtime auth context. - pub(super) fn new( - runtime: Arc, - allow_anonymous: bool, - token_ttl: Duration, - ) -> Self { + pub(super) fn new(runtime: Arc, allow_anonymous: bool, token_ttl: Duration) -> Self { Self { runtime, allow_anonymous, diff --git a/beacon-api/src/flight_sql/service.rs b/beacon-api/src/flight_sql/service.rs index 42378f58..620b727c 100644 --- a/beacon-api/src/flight_sql/service.rs +++ b/beacon-api/src/flight_sql/service.rs @@ -84,7 +84,10 @@ impl BeaconFlightSqlService { ) -> Result { let stream = self .runtime - .run_query(beacon_core::query::Query::sql(sql.clone()), auth.identity.clone()) + .run_query( + beacon_core::query::Query::sql(sql.clone()), + auth.identity.clone(), + ) .await .map_err(to_internal_status)? .into_record_stream() @@ -406,11 +409,14 @@ impl FlightSqlService for BeaconFlightSqlService { } else { let stream = self .runtime - .run_query(beacon_core::query::Query::sql(query.query.clone()), auth.identity.clone()) - .await - .map_err(to_internal_status)? - .into_record_stream() - .map_err(to_internal_status)?; + .run_query( + beacon_core::query::Query::sql(query.query.clone()), + auth.identity.clone(), + ) + .await + .map_err(to_internal_status)? + .into_record_stream() + .map_err(to_internal_status)?; encode_schema(stream.schema().as_ref())? }; diff --git a/beacon-api/src/flight_sql/storage.rs b/beacon-api/src/flight_sql/storage.rs index 78de4c3f..8e8315f6 100644 --- a/beacon-api/src/flight_sql/storage.rs +++ b/beacon-api/src/flight_sql/storage.rs @@ -96,4 +96,4 @@ impl SqlHandleStore { let now = Instant::now(); statements.retain(|_, statement| statement.expires_at > now); } -} \ No newline at end of file +} diff --git a/beacon-api/src/flight_sql/tests.rs b/beacon-api/src/flight_sql/tests.rs index 16f27e76..82176537 100644 --- a/beacon-api/src/flight_sql/tests.rs +++ b/beacon-api/src/flight_sql/tests.rs @@ -15,7 +15,13 @@ async fn spawn_server(allow_anonymous: bool) -> (SocketAddr, tokio::task::JoinHa drop(tmp); let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap(); - let runtime = Arc::new(beacon_core::runtime::Runtime::new_with_in_memory_auth(std::sync::Arc::new(beacon_config::Config::load().unwrap())).await.unwrap()); + let runtime = Arc::new( + beacon_core::runtime::Runtime::new_with_in_memory_auth(std::sync::Arc::new( + beacon_config::Config::load().unwrap(), + )) + .await + .unwrap(), + ); let service = BeaconFlightSqlService::new_with_options(runtime, allow_anonymous).unwrap(); let handle = tokio::spawn(async move { @@ -55,7 +61,13 @@ async fn spawn_server_with_runtime( drop(tmp); let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap(); - let runtime = Arc::new(beacon_core::runtime::Runtime::new_with_in_memory_auth(std::sync::Arc::new(beacon_config::Config::load().unwrap())).await.unwrap()); + let runtime = Arc::new( + beacon_core::runtime::Runtime::new_with_in_memory_auth(std::sync::Arc::new( + beacon_config::Config::load().unwrap(), + )) + .await + .unwrap(), + ); let service = BeaconFlightSqlService::new_with_options(runtime.clone(), allow_anonymous).unwrap(); @@ -86,7 +98,10 @@ async fn run_sql_rows( sql: &str, ) -> Vec { runtime - .run_query(beacon_core::query::Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + beacon_core::query::Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await .expect("query should run") .into_record_stream() @@ -121,7 +136,11 @@ async fn federated_remote_table_pushes_down_and_streams() { let remote_obs = format!("remote_obs_{suffix}"); // Seed the "remote" table. - run_sql_rows(&runtime, &format!("CREATE TABLE {obs} (id BIGINT, val DOUBLE)")).await; + run_sql_rows( + &runtime, + &format!("CREATE TABLE {obs} (id BIGINT, val DOUBLE)"), + ) + .await; run_sql_rows( &runtime, &format!("INSERT INTO {obs} VALUES (1, 10.0), (2, 20.0), (3, 30.0)"), diff --git a/beacon-auth/src/basic.rs b/beacon-auth/src/basic.rs index 47e4f85f..5b33bb98 100644 --- a/beacon-auth/src/basic.rs +++ b/beacon-auth/src/basic.rs @@ -212,7 +212,10 @@ mod tests { store.grant_role("alice", "reader").unwrap(); store.grant_role("alice", "writer").unwrap(); store.revoke_role("alice", "writer").unwrap(); - assert_eq!(store.verify("alice", "secret").unwrap(), vec!["reader".to_string()]); + assert_eq!( + store.verify("alice", "secret").unwrap(), + vec!["reader".to_string()] + ); store.drop_user("alice").unwrap(); assert!(!store.user_exists("alice")); diff --git a/beacon-auth/src/composite.rs b/beacon-auth/src/composite.rs index 6264c225..f0e6b4ba 100644 --- a/beacon-auth/src/composite.rs +++ b/beacon-auth/src/composite.rs @@ -45,7 +45,10 @@ impl AuthProvider for CompositeAuthProvider { #[cfg(test)] mod tests { use super::*; - use crate::{basic::BasicAuthProvider, oidc::{OidcAuthProvider, OidcConfig}}; + use crate::{ + basic::BasicAuthProvider, + oidc::{OidcAuthProvider, OidcConfig}, + }; use std::time::Duration; fn composite() -> CompositeAuthProvider { @@ -86,7 +89,10 @@ mod tests { .await .unwrap_err() .to_string(); - assert!(err.contains("token header") || err.contains("invalid"), "got: {err}"); + assert!( + err.contains("token header") || err.contains("invalid"), + "got: {err}" + ); } #[test] diff --git a/beacon-auth/src/context.rs b/beacon-auth/src/context.rs index aea14ad9..267c5d88 100644 --- a/beacon-auth/src/context.rs +++ b/beacon-auth/src/context.rs @@ -235,9 +235,9 @@ impl AuthContext { // --- User management (delegated to the provider's user directory) --- fn user_directory(&self) -> anyhow::Result> { - self.auth_provider - .user_directory() - .ok_or_else(|| anyhow::anyhow!("the active auth provider does not support user management")) + self.auth_provider.user_directory().ok_or_else(|| { + anyhow::anyhow!("the active auth provider does not support user management") + }) } /// Whether a user exists in the active provider's directory (false if it has none). @@ -334,7 +334,8 @@ mod tests { ctx.create_role("reader").unwrap(); ctx.create_user("alice", "secret").unwrap(); ctx.grant_role_to_user("alice", "reader").unwrap(); - ctx.grant("reader", PrivilegeRule::new(Privilege::Select, None)).unwrap(); + ctx.grant("reader", PrivilegeRule::new(Privilege::Select, None)) + .unwrap(); ctx.deny( "reader", PrivilegeRule::new( @@ -344,7 +345,10 @@ mod tests { ) .unwrap(); - let identity = ctx.authenticate(&Credential::basic("alice", "secret")).await.unwrap(); + let identity = ctx + .authenticate(&Credential::basic("alice", "secret")) + .await + .unwrap(); assert_eq!(identity.username, "alice"); assert_eq!(identity.roles, vec!["reader".to_string()]); assert!(!identity.is_super_user); @@ -365,13 +369,19 @@ mod tests { async fn only_the_configured_credential_is_super_user() { let ctx = context_with_super_user(); // The configured credential authenticates as the super-user. - let admin = ctx.authenticate(&Credential::basic("root", "secret")).await.unwrap(); + let admin = ctx + .authenticate(&Credential::basic("root", "secret")) + .await + .unwrap(); assert!(admin.is_super_user); assert_eq!(admin.username, "root"); assert!(admin.roles.is_empty()); // The right username with the wrong password is not super (and not in the store either). - assert!(ctx.authenticate(&Credential::basic("root", "wrong")).await.is_err()); + assert!(ctx + .authenticate(&Credential::basic("root", "wrong")) + .await + .is_err()); } #[tokio::test] @@ -381,9 +391,13 @@ mod tests { ctx.create_role("reader").unwrap(); ctx.create_user("alice", "pw").unwrap(); ctx.grant_role_to_user("alice", "reader").unwrap(); - ctx.grant("reader", PrivilegeRule::new(Privilege::Select, None)).unwrap(); + ctx.grant("reader", PrivilegeRule::new(Privilege::Select, None)) + .unwrap(); - let identity = ctx.authenticate(&Credential::basic("alice", "pw")).await.unwrap(); + let identity = ctx + .authenticate(&Credential::basic("alice", "pw")) + .await + .unwrap(); assert!(!identity.is_super_user); // Write/management privileges cannot be granted to a role at all — roles are read-only. @@ -396,7 +410,8 @@ mod tests { Privilege::All, ] { assert!( - ctx.grant("reader", PrivilegeRule::new(privilege, None)).is_err(), + ctx.grant("reader", PrivilegeRule::new(privilege, None)) + .is_err(), "granting {privilege} to a role must be rejected" ); } @@ -426,7 +441,8 @@ mod tests { let mut ctx = admin_context(); ctx.create_role("public").unwrap(); ctx.create_user(ANONYMOUS_USERNAME, "").unwrap(); - ctx.grant_role_to_user(ANONYMOUS_USERNAME, "public").unwrap(); + ctx.grant_role_to_user(ANONYMOUS_USERNAME, "public") + .unwrap(); ctx.set_anonymous_user(ANONYMOUS_USERNAME); assert!(ctx.anonymous_enabled()); @@ -448,6 +464,11 @@ mod tests { // The super-user is not a stored user — `user_exists` is false even though it authenticates. let ctx = context_with_super_user(); assert!(!ctx.user_exists("root")); - assert!(ctx.authenticate(&Credential::basic("root", "secret")).await.unwrap().is_super_user); + assert!( + ctx.authenticate(&Credential::basic("root", "secret")) + .await + .unwrap() + .is_super_user + ); } } diff --git a/beacon-auth/src/oidc.rs b/beacon-auth/src/oidc.rs index 0afdbaed..d6105d77 100644 --- a/beacon-auth/src/oidc.rs +++ b/beacon-auth/src/oidc.rs @@ -127,7 +127,10 @@ impl OidcAuthProvider { let username = claim_at(&claims, &self.config.username_claim) .and_then(Value::as_str) .ok_or_else(|| { - anyhow::anyhow!("token is missing the '{}' claim", self.config.username_claim) + anyhow::anyhow!( + "token is missing the '{}' claim", + self.config.username_claim + ) })? .to_string(); @@ -157,7 +160,8 @@ impl AuthProvider for OidcAuthProvider { /// Resolves a dotted claim path (e.g. `realm_access.roles`) within a claims object. fn claim_at<'a>(claims: &'a Value, path: &str) -> Option<&'a Value> { - path.split('.').try_fold(claims, |value, segment| value.get(segment)) + path.split('.') + .try_fold(claims, |value, segment| value.get(segment)) } /// Extracts role names from a claim, accepting either an array of strings or a single @@ -188,7 +192,10 @@ mod tests { Some("alice") ); let roles = claim_at(&claims, "realm_access.roles").map(roles_from_claim); - assert_eq!(roles, Some(vec!["reader".to_string(), "writer".to_string()])); + assert_eq!( + roles, + Some(vec!["reader".to_string(), "writer".to_string()]) + ); assert!(claim_at(&claims, "missing.path").is_none()); } diff --git a/beacon-auth/src/password.rs b/beacon-auth/src/password.rs index 657ee3bf..550f00a6 100644 --- a/beacon-auth/src/password.rs +++ b/beacon-auth/src/password.rs @@ -41,7 +41,10 @@ mod tests { fn hashes_are_salted_and_not_plaintext() { let a = hash_password("same").unwrap(); let b = hash_password("same").unwrap(); - assert_ne!(a, b, "salt should make identical passwords hash differently"); + assert_ne!( + a, b, + "salt should make identical passwords hash differently" + ); assert!(!a.contains("same")); } diff --git a/beacon-auth/src/role.rs b/beacon-auth/src/role.rs index b85fc9db..63c41a84 100644 --- a/beacon-auth/src/role.rs +++ b/beacon-auth/src/role.rs @@ -138,10 +138,18 @@ pub trait RoleStore: std::fmt::Debug + Send + Sync { fn load_roles(&self) -> anyhow::Result>; fn persist_create_role(&self, name: &str) -> anyhow::Result<()>; fn persist_drop_role(&self, name: &str) -> anyhow::Result<()>; - fn persist_insert_rule(&self, role: &str, is_deny: bool, rule: &PrivilegeRule) - -> anyhow::Result<()>; - fn persist_remove_rule(&self, role: &str, is_deny: bool, rule: &PrivilegeRule) - -> anyhow::Result<()>; + fn persist_insert_rule( + &self, + role: &str, + is_deny: bool, + rule: &PrivilegeRule, + ) -> anyhow::Result<()>; + fn persist_remove_rule( + &self, + role: &str, + is_deny: bool, + rule: &PrivilegeRule, + ) -> anyhow::Result<()>; } /// In-memory registry of roles, with interior mutability for SQL-driven management. @@ -268,21 +276,22 @@ impl RoleProvider { target: &ConcreteTarget, ) -> bool { let registry = self.roles.read(); - let matched: Vec<&Role> = roles - .iter() - .filter_map(|name| registry.get(name)) - .collect(); - - let denied = matched - .iter() - .any(|role| role.denies.iter().any(|rule| rule.matches(privilege, target))); + let matched: Vec<&Role> = roles.iter().filter_map(|name| registry.get(name)).collect(); + + let denied = matched.iter().any(|role| { + role.denies + .iter() + .any(|rule| rule.matches(privilege, target)) + }); if denied { return false; } - matched - .iter() - .any(|role| role.grants.iter().any(|rule| rule.matches(privilege, target))) + matched.iter().any(|role| { + role.grants + .iter() + .any(|rule| rule.matches(privilege, target)) + }) } } diff --git a/beacon-auth/src/sqlite.rs b/beacon-auth/src/sqlite.rs index 123df39c..2fcb44b8 100644 --- a/beacon-auth/src/sqlite.rs +++ b/beacon-auth/src/sqlite.rs @@ -401,7 +401,10 @@ mod tests { assert!(store.create_user("bob", "pw").is_err()); store.grant_role("bob", "writer").unwrap(); - assert_eq!(store.verify("bob", "pw").unwrap(), vec!["writer".to_string()]); + assert_eq!( + store.verify("bob", "pw").unwrap(), + vec!["writer".to_string()] + ); assert!(store.verify("bob", "wrong").is_err()); store.revoke_role("bob", "writer").unwrap(); diff --git a/beacon-common/src/cf_time.rs b/beacon-common/src/cf_time.rs index 589c21d1..724573bd 100644 --- a/beacon-common/src/cf_time.rs +++ b/beacon-common/src/cf_time.rs @@ -208,18 +208,22 @@ fn parse_cf_time_epoch_julian(units: &str) -> Result<(Epoch, Unit)> { })?; // Parse the date into year, month, day - let mut year: i32 = caps["year"] - .parse() - .map_err(|e| CommonError::CfTime(format!("Invalid year in units string: {units}. Error: {e}")))?; - let month_num: u32 = caps["month"] - .parse() - .map_err(|e| CommonError::CfTime(format!("Invalid month in units string: {units}. Error: {e}")))?; + let mut year: i32 = caps["year"].parse().map_err(|e| { + CommonError::CfTime(format!("Invalid year in units string: {units}. Error: {e}")) + })?; + let month_num: u32 = caps["month"].parse().map_err(|e| { + CommonError::CfTime(format!( + "Invalid month in units string: {units}. Error: {e}" + )) + })?; let month = julian::Month::try_from(month_num).map_err(|e| { - CommonError::CfTime(format!("Invalid month in units string: {units}. Error: {e:?}")) + CommonError::CfTime(format!( + "Invalid month in units string: {units}. Error: {e:?}" + )) + })?; + let day: u32 = caps["day"].parse().map_err(|e| { + CommonError::CfTime(format!("Invalid day in units string: {units}. Error: {e}")) })?; - let day: u32 = caps["day"] - .parse() - .map_err(|e| CommonError::CfTime(format!("Invalid day in units string: {units}. Error: {e}")))?; let jul_cal = julian::Calendar::JULIAN; diff --git a/beacon-common/src/file_descriptors.rs b/beacon-common/src/file_descriptors.rs index 2209f2f9..960f5872 100644 --- a/beacon-common/src/file_descriptors.rs +++ b/beacon-common/src/file_descriptors.rs @@ -8,7 +8,7 @@ pub fn max_open_fd() -> u64 { #[cfg(unix)] { - use rlimit::{Resource, getrlimit}; + use rlimit::{getrlimit, Resource}; if let Ok((soft_limit, _)) = getrlimit(Resource::NOFILE) { tracing::debug!( "Max open file descriptors (NOFILE soft limit): {}", @@ -26,6 +26,6 @@ pub fn max_open_fd() -> u64 { pub fn file_open_parallelism() -> usize { let max = max_open_fd() as usize / 2; // use half of the available file descriptors for parallelism to be safe - //Make sure max is at least 1 to avoid zero parallelism + //Make sure max is at least 1 to avoid zero parallelism std::cmp::max(max, 1) } diff --git a/beacon-common/src/super_typing.rs b/beacon-common/src/super_typing.rs index 093e9538..d454cb49 100644 --- a/beacon-common/src/super_typing.rs +++ b/beacon-common/src/super_typing.rs @@ -488,7 +488,10 @@ mod tests { None ); // Boolean still has no common type with non-numeric, non-string types. - assert_eq!(super_type_arrow(&DataType::Boolean, &DataType::Date32), None); + assert_eq!( + super_type_arrow(&DataType::Boolean, &DataType::Date32), + None + ); } fn schema(fields: &[(&str, DataType)]) -> SchemaRef { @@ -517,9 +520,18 @@ mod tests { let names: Vec<&str> = merged.fields.iter().map(|f| f.name().as_str()).collect(); assert_eq!(names, vec!["a", "b", "c"]); - assert_eq!(merged.field_with_name("a").unwrap().data_type(), &DataType::Int32); - assert_eq!(merged.field_with_name("b").unwrap().data_type(), &DataType::Float64); - assert_eq!(merged.field_with_name("c").unwrap().data_type(), &DataType::Utf8); + assert_eq!( + merged.field_with_name("a").unwrap().data_type(), + &DataType::Int32 + ); + assert_eq!( + merged.field_with_name("b").unwrap().data_type(), + &DataType::Float64 + ); + assert_eq!( + merged.field_with_name("c").unwrap().data_type(), + &DataType::Utf8 + ); // super_type_schema marks every output field nullable. assert!(merged.fields.iter().all(|f| f.is_nullable())); diff --git a/beacon-common/src/table_function.rs b/beacon-common/src/table_function.rs index 28e9feb9..24dc5262 100644 --- a/beacon-common/src/table_function.rs +++ b/beacon-common/src/table_function.rs @@ -32,7 +32,10 @@ pub trait BeaconTableFunctionImpl: TableFunctionImpl + Send + Sync { for option in options { all_datatypes.push(option.data_type().clone()); } - Signature::exact(all_datatypes, datafusion::logical_expr::Volatility::Immutable) + Signature::exact( + all_datatypes, + datafusion::logical_expr::Volatility::Immutable, + ) } fn documentation(&self) -> Option { None @@ -43,7 +46,10 @@ pub trait BeaconTableFunctionImpl: TableFunctionImpl + Send + Sync { /// /// Accepts either a single string scalar (`Utf8`/`LargeUtf8`/`Utf8View`) or a /// `List` of strings. `fn_name` is used only to build clear error messages. -pub fn parse_glob_paths_arg(args: &[Expr], fn_name: &str) -> datafusion::error::Result> { +pub fn parse_glob_paths_arg( + args: &[Expr], + fn_name: &str, +) -> datafusion::error::Result> { let Some(first) = args.first() else { return plan_err!("{fn_name} requires at least 1 argument: glob_paths : Utf8 | List"); }; @@ -116,7 +122,10 @@ mod tests { fn list_of_strings_is_accepted() { let expr = list_expr(&["a.parquet", "b.parquet"]); let paths = parse_glob_paths_arg(&[expr], "read_parquet").unwrap(); - assert_eq!(paths, vec!["a.parquet".to_string(), "b.parquet".to_string()]); + assert_eq!( + paths, + vec!["a.parquet".to_string(), "b.parquet".to_string()] + ); } #[test] diff --git a/beacon-config/src/lib.rs b/beacon-config/src/lib.rs index 407328c6..7a29ba97 100644 --- a/beacon-config/src/lib.rs +++ b/beacon-config/src/lib.rs @@ -364,10 +364,7 @@ struct RawConfig { default = "Content-Type,Authorization" )] allowed_headers: String, - #[envconfig( - from = "BEACON_CORS_EXPOSE_HEADERS", - default = "x-beacon-query-id" - )] + #[envconfig(from = "BEACON_CORS_EXPOSE_HEADERS", default = "x-beacon-query-id")] expose_headers: String, #[envconfig(from = "BEACON_CORS_ALLOWED_CREDENTIALS", default = "false")] allowed_credentials: bool, @@ -484,7 +481,9 @@ impl From for Config { default_table_engine: match raw.default_table_engine.parse() { Ok(engine) => engine, Err(e) => { - tracing::warn!("invalid BEACON_DEFAULT_TABLE_ENGINE: {e}; defaulting to lance"); + tracing::warn!( + "invalid BEACON_DEFAULT_TABLE_ENGINE: {e}; defaulting to lance" + ); TableEngine::default() } }, @@ -809,7 +808,10 @@ mod tests { fn table_engine_parses_case_insensitively_and_trims() { assert_eq!("lance".parse::(), Ok(TableEngine::Lance)); assert_eq!("ICEBERG".parse::(), Ok(TableEngine::Iceberg)); - assert_eq!(" Iceberg ".parse::(), Ok(TableEngine::Iceberg)); + assert_eq!( + " Iceberg ".parse::(), + Ok(TableEngine::Iceberg) + ); assert!("postgres".parse::().is_err()); } diff --git a/beacon-core/src/api.rs b/beacon-core/src/api.rs index ae44d625..1f9fdf08 100644 --- a/beacon-core/src/api.rs +++ b/beacon-core/src/api.rs @@ -3,12 +3,12 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; +use crate::metrics::ConsolidatedMetrics; use arrow::datatypes::{Field, Schema}; use beacon_data_lake::crawler::{CrawlReport, CrawlerDefinition, TableNaming}; use beacon_datafusion_ext::format_ext::DatasetMetadata; use beacon_datafusion_ext::table_ext::TableDefinition; use beacon_functions::function_doc::FunctionDoc; -use crate::metrics::ConsolidatedMetrics; use serde_json::{Map, Value}; use utoipa::ToSchema; @@ -328,7 +328,9 @@ impl TryFrom> for TableConfigView { /// How a crawler turns a discovered group of files into a table name. Mirrors the /// data-lake [`TableNaming`] so the API surface need not depend on its internals. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, ToSchema)] +#[derive( + Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, ToSchema, +)] #[serde(rename_all = "snake_case")] pub enum TableNamingView { /// Use the leaf component of the group's base prefix (`argo/floats` -> `floats`). @@ -553,7 +555,10 @@ mod table_config_redaction_tests { assert!(!json.contains("super-secret-password")); // The encrypted material (ciphertext/nonce) must not leak either. assert!(!json.contains("ciphertext")); - assert_eq!(view.config.get("secret"), Some(&Value::String("***".to_string()))); + assert_eq!( + view.config.get("secret"), + Some(&Value::String("***".to_string())) + ); // Non-secret connection options remain visible. assert!(json.contains("db.internal")); } diff --git a/beacon-core/src/dataset_uploads.rs b/beacon-core/src/dataset_uploads.rs index b7b78950..0e6a833a 100644 --- a/beacon-core/src/dataset_uploads.rs +++ b/beacon-core/src/dataset_uploads.rs @@ -115,12 +115,7 @@ impl UploadManager { /// On an object-store failure the whole session is aborted and removed, since /// a partially-written multipart upload cannot be safely resumed; the client /// must restart the upload. - pub async fn put_part( - &self, - id: Uuid, - part_number: u32, - data: Bytes, - ) -> Result<(), FileError> { + pub async fn put_part(&self, id: Uuid, part_number: u32, data: Bytes) -> Result<(), FileError> { let session = self.session(id)?; let mut guard = session.lock().await; @@ -205,7 +200,9 @@ impl UploadManager { /// Spawn the background task that aborts sessions idle longer than `ttl`. It /// stops once the manager is dropped (the `Weak` no longer upgrades). fn spawn_sweeper(manager: Weak, ttl: Duration) { - let interval = ttl.min(MIN_SWEEP_INTERVAL.max(ttl / 4)).max(MIN_SWEEP_INTERVAL); + let interval = ttl + .min(MIN_SWEEP_INTERVAL.max(ttl / 4)) + .max(MIN_SWEEP_INTERVAL); tokio::spawn(async move { loop { tokio::time::sleep(interval).await; @@ -264,8 +261,12 @@ mod tests { let path = Path::from("big/data.parquet"); let id = mgr.initiate(&store, path.clone(), 0).await.unwrap(); - mgr.put_part(id, 1, Bytes::from_static(b"aaaa")).await.unwrap(); - mgr.put_part(id, 2, Bytes::from_static(b"bbbb")).await.unwrap(); + mgr.put_part(id, 1, Bytes::from_static(b"aaaa")) + .await + .unwrap(); + mgr.put_part(id, 2, Bytes::from_static(b"bbbb")) + .await + .unwrap(); let res = mgr.complete(id).await.unwrap(); assert_eq!(res.size, 8); assert_eq!(res.path, "big/data.parquet"); @@ -282,10 +283,16 @@ mod tests { .initiate(&store, Path::from("a.parquet"), 0) .await .unwrap(); - mgr.put_part(id, 1, Bytes::from_static(b"one")).await.unwrap(); + mgr.put_part(id, 1, Bytes::from_static(b"one")) + .await + .unwrap(); // Re-send part 1 (e.g. a lost response): accepted as a no-op. - mgr.put_part(id, 1, Bytes::from_static(b"one")).await.unwrap(); - mgr.put_part(id, 2, Bytes::from_static(b"two")).await.unwrap(); + mgr.put_part(id, 1, Bytes::from_static(b"one")) + .await + .unwrap(); + mgr.put_part(id, 2, Bytes::from_static(b"two")) + .await + .unwrap(); let res = mgr.complete(id).await.unwrap(); assert_eq!(res.size, 6); } @@ -298,11 +305,16 @@ mod tests { .initiate(&store, Path::from("a.parquet"), 0) .await .unwrap(); - mgr.put_part(id, 1, Bytes::from_static(b"one")).await.unwrap(); + mgr.put_part(id, 1, Bytes::from_static(b"one")) + .await + .unwrap(); // Skipping part 2 → 3 is a gap. assert!(matches!( mgr.put_part(id, 3, Bytes::from_static(b"three")).await, - Err(FileError::PartOutOfOrder { got: 3, expected: 2 }) + Err(FileError::PartOutOfOrder { + got: 3, + expected: 2 + }) )); } @@ -328,7 +340,9 @@ mod tests { .initiate(&store, Path::from("a.parquet"), 5) .await .unwrap(); - mgr.put_part(id, 1, Bytes::from_static(b"abc")).await.unwrap(); + mgr.put_part(id, 1, Bytes::from_static(b"abc")) + .await + .unwrap(); // 3 + 4 = 7 > cap of 5 → rejected, session torn down. assert!(matches!( mgr.put_part(id, 2, Bytes::from_static(b"defg")).await, diff --git a/beacon-core/src/extensions.rs b/beacon-core/src/extensions.rs index b87ce15a..a4035e5b 100644 --- a/beacon-core/src/extensions.rs +++ b/beacon-core/src/extensions.rs @@ -210,18 +210,16 @@ impl TableExtensions { pub fn set_kind(&mut self, kind: &str, json: &str) -> anyhow::Result<()> { match kind.to_ascii_lowercase().as_str() { "mcp" => { - self.mcp = Some( - serde_json::from_str(json).context("invalid 'mcp' extension payload")?, - ); + self.mcp = + Some(serde_json::from_str(json).context("invalid 'mcp' extension payload")?); } "preset" => { - self.preset = Some( - serde_json::from_str(json).context("invalid 'preset' extension payload")?, - ); + self.preset = + Some(serde_json::from_str(json).context("invalid 'preset' extension payload")?); + } + other => { + anyhow::bail!("unknown extension kind '{other}'; expected one of: mcp, preset") } - other => anyhow::bail!( - "unknown extension kind '{other}'; expected one of: mcp, preset" - ), } Ok(()) } @@ -231,9 +229,9 @@ impl TableExtensions { match kind.to_ascii_lowercase().as_str() { "mcp" => self.mcp = None, "preset" => self.preset = None, - other => anyhow::bail!( - "unknown extension kind '{other}'; expected one of: mcp, preset" - ), + other => { + anyhow::bail!("unknown extension kind '{other}'; expected one of: mcp, preset") + } } Ok(()) } @@ -338,7 +336,13 @@ fn ensure_column(schema: &Schema, column: &str) -> anyhow::Result<()> { pub fn show_extensions_arrow_schema() -> SchemaRef { static SCHEMA: OnceLock = OnceLock::new(); SCHEMA - .get_or_init(|| Arc::new(Schema::new(vec![Field::new("extensions", DataType::Utf8, false)]))) + .get_or_init(|| { + Arc::new(Schema::new(vec![Field::new( + "extensions", + DataType::Utf8, + false, + )])) + }) .clone() } @@ -362,8 +366,9 @@ pub async fn get_table_extensions( ) -> anyhow::Result { anyhow::ensure!(ctx.table_exist(name)?, "table '{name}' not found"); match persistence(ctx).load_table_extensions_json(name).await? { - Some(json) => Ok(serde_json::from_str(&json) - .context("stored table extensions are not valid")?), + Some(json) => { + Ok(serde_json::from_str(&json).context("stored table extensions are not valid")?) + } None => Ok(TableExtensions::default()), } } @@ -410,10 +415,7 @@ pub async fn set_table_extensions( } /// Remove all extensions for a table. -pub async fn delete_table_extensions( - ctx: &Arc, - name: &str, -) -> anyhow::Result<()> { +pub async fn delete_table_extensions(ctx: &Arc, name: &str) -> anyhow::Result<()> { anyhow::ensure!(ctx.table_exist(name)?, "table '{name}' not found"); persistence(ctx).remove_table_extensions_json(name).await?; Ok(()) @@ -517,7 +519,11 @@ mod tests { r#"{"presets":[{"name":"p","filters":[{"column":"lat","op":"between","value":5}]}]}"#, ) .unwrap(); - assert!(ext.validate(&schema()).unwrap_err().to_string().contains("between")); + assert!(ext + .validate(&schema()) + .unwrap_err() + .to_string() + .contains("between")); } #[test] @@ -528,7 +534,11 @@ mod tests { r#"{"presets":[{"name":"p","filters":[]},{"name":"p","filters":[]}]}"#, ) .unwrap(); - assert!(ext.validate(&schema()).unwrap_err().to_string().contains("duplicate preset")); + assert!(ext + .validate(&schema()) + .unwrap_err() + .to_string() + .contains("duplicate preset")); } #[test] @@ -548,9 +558,16 @@ mod tests { #[test] fn mcp_exposed_columns_must_exist() { let mut ext = TableExtensions::default(); - ext.set_kind("mcp", r#"{"enabled":true,"exposed_columns":["lat","ghost"]}"#) - .unwrap(); - assert!(ext.validate(&schema()).unwrap_err().to_string().contains("does not exist")); + ext.set_kind( + "mcp", + r#"{"enabled":true,"exposed_columns":["lat","ghost"]}"#, + ) + .unwrap(); + assert!(ext + .validate(&schema()) + .unwrap_err() + .to_string() + .contains("does not exist")); } #[test] @@ -565,7 +582,10 @@ mod tests { assert!(ext.validate(&schema()).is_ok()); let cols = ext.mcp.as_ref().unwrap().exposed_columns.as_ref().unwrap(); assert_eq!((cols[0].name(), cols[0].description()), ("lat", None)); - assert_eq!((cols[1].name(), cols[1].description()), ("depth", Some("meters"))); + assert_eq!( + (cols[1].name(), cols[1].description()), + ("depth", Some("meters")) + ); // A documented column with an unknown key is rejected (deny_unknown_fields). let mut bad = TableExtensions::default(); assert!(bad diff --git a/beacon-core/src/lib.rs b/beacon-core/src/lib.rs index b6f6b1a8..cd2ae82a 100644 --- a/beacon-core/src/lib.rs +++ b/beacon-core/src/lib.rs @@ -1,7 +1,7 @@ pub mod api; -pub mod extensions; pub mod dataset_files; pub mod dataset_uploads; +pub mod extensions; pub mod metrics; pub mod parser; pub mod query; diff --git a/beacon-core/src/metrics.rs b/beacon-core/src/metrics.rs index 4f9d1e00..9f5891df 100644 --- a/beacon-core/src/metrics.rs +++ b/beacon-core/src/metrics.rs @@ -6,7 +6,7 @@ use std::{ collections::HashMap, - sync::{Arc, atomic::AtomicU64}, + sync::{atomic::AtomicU64, Arc}, }; use datafusion::{logical_expr::LogicalPlan, physical_plan::ExecutionPlan}; @@ -249,7 +249,10 @@ fn node_to_pg_json(plan: &dyn ExecutionPlan) -> serde_json::Value { if name == "output_rows" || name == "elapsed_compute" { continue; } - extras.insert(name.to_string(), serde_json::json!(metric.value().as_usize())); + extras.insert( + name.to_string(), + serde_json::json!(metric.value().as_usize()), + ); } if !extras.is_empty() { node["Extras"] = serde_json::Value::Object(extras); diff --git a/beacon-core/src/parser/beacon_parser.rs b/beacon-core/src/parser/beacon_parser.rs index 4437a179..68e8f1ce 100644 --- a/beacon-core/src/parser/beacon_parser.rs +++ b/beacon-core/src/parser/beacon_parser.rs @@ -95,11 +95,15 @@ impl<'a> BeaconParser<'a> { } fn is_create_crawler(&self) -> bool { - self.is_keyword_then_crawler(|t| matches!(t, Token::Word(w) if w.keyword == Keyword::CREATE)) + self.is_keyword_then_crawler( + |t| matches!(t, Token::Word(w) if w.keyword == Keyword::CREATE), + ) } fn is_run_crawler(&self) -> bool { - self.is_keyword_then_crawler(|t| matches!(t, Token::Word(w) if w.value.to_uppercase() == "RUN")) + self.is_keyword_then_crawler( + |t| matches!(t, Token::Word(w) if w.value.to_uppercase() == "RUN"), + ) } fn is_drop_crawler(&self) -> bool { @@ -195,7 +199,9 @@ impl<'a> BeaconParser<'a> { } fn is_drop_extension(&self) -> bool { - self.is_keyword_then_extension(|t| matches!(t, Token::Word(w) if w.keyword == Keyword::DROP)) + self.is_keyword_then_extension( + |t| matches!(t, Token::Word(w) if w.keyword == Keyword::DROP), + ) } fn is_show_extensions(&self) -> bool { @@ -360,7 +366,10 @@ impl<'a> BeaconParser<'a> { .parser .parse_object_name(false) .map_err(|e| DataFusionError::External(Box::new(e)))?; - Ok(BeaconStatement::DropIndex(DropIndexStatement { name, table })) + Ok(BeaconStatement::DropIndex(DropIndexStatement { + name, + table, + })) } /// Parse: SHOW INDEXES [ON|FROM] @@ -666,8 +675,8 @@ impl<'a> BeaconParser<'a> { /// or `ALL`. fn parse_privilege_and_target(&mut self) -> Result<(Privilege, Option)> { let privilege_str = self.parse_string_value()?; - let privilege = Privilege::from_str(&privilege_str) - .map_err(|err| DataFusionError::Plan(err))?; + let privilege = + Privilege::from_str(&privilege_str).map_err(|err| DataFusionError::Plan(err))?; let target = if self.word_at(0, "ON") { self.df_parser.parser.next_token(); // ON @@ -742,8 +751,12 @@ mod tests { #[test] fn parse_role_lifecycle_and_assignment() { - assert!(matches!(parse_auth("CREATE ROLE reader"), AuthStatement::CreateRole { role } if role == "reader")); - assert!(matches!(parse_auth("DROP ROLE reader"), AuthStatement::DropRole { role } if role == "reader")); + assert!( + matches!(parse_auth("CREATE ROLE reader"), AuthStatement::CreateRole { role } if role == "reader") + ); + assert!( + matches!(parse_auth("DROP ROLE reader"), AuthStatement::DropRole { role } if role == "reader") + ); match parse_auth("GRANT ROLE reader TO USER alice") { AuthStatement::GrantRoleToUser { role, username } => { assert_eq!(role, "reader"); @@ -763,22 +776,36 @@ mod tests { #[test] fn parse_privilege_grants_with_targets() { match parse_auth("GRANT SELECT ON PATH 'argo/**/*.nc' TO ROLE reader") { - AuthStatement::GrantPrivilege { privilege, target, role } => { + AuthStatement::GrantPrivilege { + privilege, + target, + role, + } => { assert_eq!(privilege, Privilege::Select); - assert_eq!(target, Some(PrivilegeTarget::Path("argo/**/*.nc".to_string()))); + assert_eq!( + target, + Some(PrivilegeTarget::Path("argo/**/*.nc".to_string())) + ); assert_eq!(role, "reader"); } other => panic!("unexpected: {other:?}"), } match parse_auth("GRANT SELECT ON TABLE observations TO ROLE reader") { AuthStatement::GrantPrivilege { target, .. } => { - assert_eq!(target, Some(PrivilegeTarget::Table("observations".to_string()))); + assert_eq!( + target, + Some(PrivilegeTarget::Table("observations".to_string())) + ); } other => panic!("unexpected: {other:?}"), } // No `ON` clause means the grant applies to every target. match parse_auth("GRANT ALL TO ROLE admin") { - AuthStatement::GrantPrivilege { privilege, target, role } => { + AuthStatement::GrantPrivilege { + privilege, + target, + role, + } => { assert_eq!(privilege, Privilege::All); assert_eq!(target, None); assert_eq!(role, "admin"); @@ -790,9 +817,16 @@ mod tests { #[test] fn parse_deny_and_revoke_variants() { match parse_auth("DENY SELECT ON PATH 'argo/restricted/*' TO ROLE reader") { - AuthStatement::DenyPrivilege { privilege, target, role } => { + AuthStatement::DenyPrivilege { + privilege, + target, + role, + } => { assert_eq!(privilege, Privilege::Select); - assert_eq!(target, Some(PrivilegeTarget::Path("argo/restricted/*".to_string()))); + assert_eq!( + target, + Some(PrivilegeTarget::Path("argo/restricted/*".to_string())) + ); assert_eq!(role, "reader"); } other => panic!("unexpected: {other:?}"), @@ -994,7 +1028,10 @@ mod tests { for sql in ["SET timezone = 'UTC'", "DROP TABLE t", "SHOW TABLES"] { let mut p = BeaconParser::new(sql).unwrap(); assert!( - matches!(p.parse_statement().unwrap(), BeaconStatement::DFStatement(_)), + matches!( + p.parse_statement().unwrap(), + BeaconStatement::DFStatement(_) + ), "`{sql}` should be a DataFusion statement" ); } diff --git a/beacon-core/src/parser/statement.rs b/beacon-core/src/parser/statement.rs index 486511c5..e0237612 100644 --- a/beacon-core/src/parser/statement.rs +++ b/beacon-core/src/parser/statement.rs @@ -88,12 +88,27 @@ fn escape_sql_literal(value: &str) -> String { /// Authentication and authorization management statements (users, roles, grants, denies). #[derive(Debug, Clone)] pub enum AuthStatement { - CreateUser { username: String, password: String }, - DropUser { username: String }, - CreateRole { role: String }, - DropRole { role: String }, - GrantRoleToUser { role: String, username: String }, - RevokeRoleFromUser { role: String, username: String }, + CreateUser { + username: String, + password: String, + }, + DropUser { + username: String, + }, + CreateRole { + role: String, + }, + DropRole { + role: String, + }, + GrantRoleToUser { + role: String, + username: String, + }, + RevokeRoleFromUser { + role: String, + username: String, + }, GrantPrivilege { privilege: Privilege, target: Option, @@ -128,21 +143,34 @@ impl Display for AuthStatement { AuthStatement::RevokeRoleFromUser { role, username } => { write!(f, "REVOKE ROLE {role} FROM USER {username}") } - AuthStatement::GrantPrivilege { privilege, target, role } => { + AuthStatement::GrantPrivilege { + privilege, + target, + role, + } => { write!(f, "GRANT {privilege}")?; if let Some(target) = target { write!(f, " ON {target}")?; } write!(f, " TO ROLE {role}") } - AuthStatement::DenyPrivilege { privilege, target, role } => { + AuthStatement::DenyPrivilege { + privilege, + target, + role, + } => { write!(f, "DENY {privilege}")?; if let Some(target) = target { write!(f, " ON {target}")?; } write!(f, " TO ROLE {role}") } - AuthStatement::RevokePrivilege { privilege, target, role, deny } => { + AuthStatement::RevokePrivilege { + privilege, + target, + role, + deny, + } => { write!(f, "REVOKE ")?; if *deny { write!(f, "DENY ")?; diff --git a/beacon-core/src/query/compiler.rs b/beacon-core/src/query/compiler.rs index 3c9caf41..077245c6 100644 --- a/beacon-core/src/query/compiler.rs +++ b/beacon-core/src/query/compiler.rs @@ -15,7 +15,10 @@ pub async fn compile_json_query( ) -> anyhow::Result { // The runtime config is published as a SessionConfig extension; fall back to // defaults if absent (e.g. a bare session in a unit test). - let config = session.state().config().get_extension::(); + let config = session + .state() + .config() + .get_extension::(); let enable_pushdown_projection = config .as_ref() .map(|c| c.sql.enable_pushdown_projection) @@ -36,11 +39,9 @@ pub async fn compile_json_query( all_columns.extend(select_cols); } - from.init_builder(session, Some(&all_columns)) - .await? + from.init_builder(session, Some(&all_columns)).await? } else { - from.init_builder(session, None) - .await? + from.init_builder(session, None).await? }; let session_state = session.state(); diff --git a/beacon-core/src/query/from.rs b/beacon-core/src/query/from.rs index 6efaa2ff..aa9d8a70 100644 --- a/beacon-core/src/query/from.rs +++ b/beacon-core/src/query/from.rs @@ -4,15 +4,11 @@ use std::sync::Arc; -use beacon_datafusion_ext::file_collection::FileCollection; -use beacon_arrow_odv::datafusion::OdvFormat; use beacon_arrow_csv::datafusion::CsvFormat; +use beacon_arrow_odv::datafusion::OdvFormat; +use beacon_datafusion_ext::file_collection::FileCollection; use datafusion::{ - datasource::{ - file_format::FileFormat, - listing::ListingTableUrl, - provider_as_source, - }, + datasource::{file_format::FileFormat, listing::ListingTableUrl, provider_as_source}, logical_expr::{LogicalPlanBuilder, TableSource}, prelude::SessionContext, }; @@ -309,20 +305,16 @@ mod tests { #[tokio::test] async fn csv_and_odv_build_directly() { let ctx = SessionContext::new(); - assert!( - FromFormat::Csv { - delimiter: Some(';'), - paths: vec![], - } + assert!(FromFormat::Csv { + delimiter: Some(';'), + paths: vec![], + } + .file_format(&ctx) + .await + .is_ok()); + assert!(FromFormat::Odv { paths: vec![] } .file_format(&ctx) .await - .is_ok() - ); - assert!( - FromFormat::Odv { paths: vec![] } - .file_format(&ctx) - .await - .is_ok() - ); + .is_ok()); } } diff --git a/beacon-core/src/query/output.rs b/beacon-core/src/query/output.rs index 04d39389..a6b5f3d4 100644 --- a/beacon-core/src/query/output.rs +++ b/beacon-core/src/query/output.rs @@ -5,12 +5,12 @@ use std::sync::Arc; -use beacon_arrow_netcdf::datafusion::{options::NetcdfOptions, NetCDFFormatFactory, NetcdfConfig}; -use beacon_arrow_odv::datafusion::OdvFileFormatFactory; -use beacon_arrow_odv::writer::OdvOptions; use beacon_arrow_csv::datafusion::CsvFormatFactory; use beacon_arrow_geoparquet::datafusion::{GeoParquetFormatFactory, GeoParquetOptions}; use beacon_arrow_ipc::datafusion::ArrowFormatFactory; +use beacon_arrow_netcdf::datafusion::{options::NetcdfOptions, NetCDFFormatFactory, NetcdfConfig}; +use beacon_arrow_odv::datafusion::OdvFileFormatFactory; +use beacon_arrow_odv::writer::OdvOptions; use beacon_arrow_parquet::datafusion::ParquetFormatFactory; use datafusion::{ common::file_options::file_type::FileType, diff --git a/beacon-core/src/query_result.rs b/beacon-core/src/query_result.rs index da0e2c8a..f5c5c91c 100644 --- a/beacon-core/src/query_result.rs +++ b/beacon-core/src/query_result.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, sync::Arc}; -use arrow::datatypes::SchemaRef; use crate::metrics::{ConsolidatedMetrics, MetricsTracker}; +use arrow::datatypes::SchemaRef; use datafusion::execution::SendableRecordBatchStream; use futures::Stream; use parking_lot::Mutex; diff --git a/beacon-core/src/runtime.rs b/beacon-core/src/runtime.rs index e5d9524a..a3124296 100644 --- a/beacon-core/src/runtime.rs +++ b/beacon-core/src/runtime.rs @@ -2,6 +2,7 @@ use std::{collections::HashMap, sync::Arc}; +use crate::metrics::{ConsolidatedMetrics, MetricsTracker}; use arrow::{ array::AsArray, datatypes::{SchemaRef, UInt64Type}, @@ -15,7 +16,6 @@ use beacon_datafusion_ext::{ stats_cache::beacon_file_statistics_cache, }; use beacon_functions::function_doc::FunctionDoc; -use crate::metrics::{ConsolidatedMetrics, MetricsTracker}; use datafusion::{ catalog::TableFunctionImpl, execution::{ @@ -29,8 +29,8 @@ use parking_lot::Mutex; use crate::{ api::{ - CrawlReportView, CrawlerView, CreateCrawlerRequest, CreateExternalTableRequest, DatasetInfo, - FunctionInfo, QueryMetricsView, QueryRequest, SchemaView, TableConfigView, + CrawlReportView, CrawlerView, CreateCrawlerRequest, CreateExternalTableRequest, + DatasetInfo, FunctionInfo, QueryMetricsView, QueryRequest, SchemaView, TableConfigView, }, parser::{beacon_parser::BeaconParser, statement::BeaconStatement}, query_result::{ArrowOutputStream, QueryOutput, QueryOutputFile, QueryResult}, @@ -189,8 +189,7 @@ impl Runtime { // Build the crawler manager once the data lake and tables exist, then publish // it through the handle so `CREATE/RUN/DROP CRAWLER` actions can reach it. - let events_available = - config.storage.enable_fs_events || config.storage.enable_s3_events; + let events_available = config.storage.enable_fs_events || config.storage.enable_s3_events; let crawler_manager = beacon_data_lake::crawler::CrawlerManager::new( session_ctx.clone(), file_formats.clone(), @@ -222,9 +221,7 @@ impl Runtime { /// /// State persists across restarts, so the bootstrap is idempotent: existing roles/users are /// left in place rather than re-created. - fn init_auth( - config: &beacon_config::Config, - ) -> anyhow::Result> { + fn init_auth(config: &beacon_config::Config) -> anyhow::Result> { let store = beacon_auth::SqliteStore::open(beacon_config::USERS_DIR.join("directory.db"))?; let role_provider = beacon_auth::RoleProvider::with_persistence(store.clone())?; let local: Arc = @@ -252,7 +249,10 @@ impl Runtime { username_claim: config.oidc.username_claim.clone(), jwks_cache_ttl: std::time::Duration::from_secs(config.oidc.jwks_cache_ttl_secs), }); - Arc::new(beacon_auth::CompositeAuthProvider::new(local, Arc::new(oidc))) + Arc::new(beacon_auth::CompositeAuthProvider::new( + local, + Arc::new(oidc), + )) } /// Builds an ephemeral, non-persistent auth context backed by the in-memory basic-auth provider. @@ -484,7 +484,8 @@ impl Runtime { (e.g. SELECT); this statement produces no result set to export" ); } - self.run_query_to_file(plan, output, query_id, query_json).await + self.run_query_to_file(plan, output, query_id, query_json) + .await } None => self.run_query_to_stream(plan, query_id, query_json).await, } @@ -524,7 +525,11 @@ impl Runtime { // `Output::parse` wraps the (already validated) plan in a `COPY TO` the // temp file; this COPY is beacon-generated, so it is not re-validated. let (copy_plan, output_file) = output - .parse(self.session_ctx.as_ref(), &self.config.storage.tmp_dir, plan) + .parse( + self.session_ctx.as_ref(), + &self.config.storage.tmp_dir, + plan, + ) .await?; let output_file = QueryOutputFile::from(output_file); @@ -566,15 +571,12 @@ impl Runtime { /// Lower a SQL statement (SELECT, DDL/DML, or a beacon custom statement) to a /// `LogicalPlan`. The result is validated in `run_query` before execution. - async fn lower_sql( - &self, - sql: &str, - ) -> anyhow::Result { + async fn lower_sql(&self, sql: &str) -> anyhow::Result { match Self::parse_beacon_statement(sql)? { BeaconStatement::Auth(statement) => Ok(crate::statement_plan::auth_plan(statement)), - BeaconStatement::CreateMaterializedView(statement) => { - Ok(crate::statement_plan::create_materialized_view_plan(statement)) - } + BeaconStatement::CreateMaterializedView(statement) => Ok( + crate::statement_plan::create_materialized_view_plan(statement), + ), BeaconStatement::Refresh(statement) => { Ok(crate::statement_plan::refresh_plan(statement)) } @@ -818,8 +820,13 @@ impl Runtime { } pub async fn list_table_config(&self, table_name: String) -> Option { - let provider = self.session_ctx.table_provider(table_name.as_str()).await.ok()?; - let config = beacon_data_lake::definition_from_provider(&table_name, provider.as_ref()).ok()?; + let provider = self + .session_ctx + .table_provider(table_name.as_str()) + .await + .ok()?; + let config = + beacon_data_lake::definition_from_provider(&table_name, provider.as_ref()).ok()?; match TableConfigView::try_from(config) { Ok(config) => Some(config), Err(error) => { @@ -981,11 +988,14 @@ impl Runtime { req: CreateExternalTableRequest, ) -> anyhow::Result<()> { let sql = build_create_external_table_sql(&req)?; - self.run_query(crate::query::Query::sql(sql), beacon_auth::AuthIdentity::system()) - .await? - .into_record_stream()? - .try_collect::>() - .await?; + self.run_query( + crate::query::Query::sql(sql), + beacon_auth::AuthIdentity::system(), + ) + .await? + .into_record_stream()? + .try_collect::>() + .await?; Ok(()) } @@ -1026,8 +1036,14 @@ impl Runtime { let path = crate::dataset_files::validate_dataset_path(raw_path)?; crate::dataset_files::validate_extension(&path, &self.dataset_upload_extensions())?; let max_bytes = self.config.storage.max_upload_bytes; - crate::dataset_files::upload_dataset(&self.datasets_store, &path, overwrite, max_bytes, body) - .await + crate::dataset_files::upload_dataset( + &self.datasets_store, + &path, + overwrite, + max_bytes, + body, + ) + .await } /// Begin a chunked (resumable) upload for a large file. Validates the path and @@ -1218,7 +1234,10 @@ mod materialized_view_tests { sql: &str, ) -> anyhow::Result> { let batches = runtime - .run_query(crate::query::Query::sql(sql.to_string()), beacon_auth::AuthIdentity::system()) + .run_query( + crate::query::Query::sql(sql.to_string()), + beacon_auth::AuthIdentity::system(), + ) .await? .into_record_stream()? .try_collect::>() @@ -1228,7 +1247,11 @@ mod materialized_view_tests { #[tokio::test(flavor = "multi_thread")] async fn materialized_view_create_query_refresh_and_drop() { - let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new(beacon_config::Config::load().unwrap())).await.expect("runtime should start"); + let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new( + beacon_config::Config::load().unwrap(), + )) + .await + .expect("runtime should start"); let suffix = uuid::Uuid::new_v4().simple(); let mv = format!("mv_test_{suffix}"); @@ -1300,7 +1323,11 @@ mod materialized_view_tests { #[tokio::test(flavor = "multi_thread")] async fn materialized_view_handles_zero_row_result() { - let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new(beacon_config::Config::load().unwrap())).await.expect("runtime should start"); + let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new( + beacon_config::Config::load().unwrap(), + )) + .await + .expect("runtime should start"); let mv = format!("mv_empty_{}", uuid::Uuid::new_v4().simple()); // A query with no rows must still create a queryable, Parquet-backed view. @@ -1343,7 +1370,10 @@ mod client_query_tests { async fn run_sql(runtime: &Runtime, sql: &str) { runtime - .run_query(crate::query::Query::sql(sql.to_string()), beacon_auth::AuthIdentity::system()) + .run_query( + crate::query::Query::sql(sql.to_string()), + beacon_auth::AuthIdentity::system(), + ) .await .expect("sql should run") .into_record_stream() @@ -1364,12 +1394,24 @@ mod client_query_tests { /// same pipeline as `run_sql`, returning a streamed result. #[tokio::test(flavor = "multi_thread")] async fn json_query_runs_through_unified_path() { - let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new(beacon_config::Config::load().unwrap())).await.expect("runtime should start"); + let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new( + beacon_config::Config::load().unwrap(), + )) + .await + .expect("runtime should start"); let suffix = uuid::Uuid::new_v4().simple(); let table = format!("json_q_{suffix}"); - run_sql(&runtime, &format!("CREATE TABLE {table} (a BIGINT, b BIGINT)")).await; - run_sql(&runtime, &format!("INSERT INTO {table} VALUES (1, 2), (3, 4)")).await; + run_sql( + &runtime, + &format!("CREATE TABLE {table} (a BIGINT, b BIGINT)"), + ) + .await; + run_sql( + &runtime, + &format!("INSERT INTO {table} VALUES (1, 2), (3, 4)"), + ) + .await; let batches = runtime .run_query( @@ -1399,7 +1441,11 @@ mod client_query_tests { let suffix = uuid::Uuid::new_v4().simple(); let table = format!("ext_{suffix}"); - run_sql(&runtime, &format!("CREATE TABLE {table} (lat BIGINT, depth BIGINT)")).await; + run_sql( + &runtime, + &format!("CREATE TABLE {table} (lat BIGINT, depth BIGINT)"), + ) + .await; // SET a preset via SQL, then read it back through the typed API. run_sql( @@ -1438,7 +1484,10 @@ mod client_query_tests { .downcast_ref::() .expect("extensions column is Utf8") .value(0); - assert!(json.contains("shallow"), "SHOW output should include the preset: {json}"); + assert!( + json.contains("shallow"), + "SHOW output should include the preset: {json}" + ); // An extension over a non-existent column is rejected by validation. let rejected = try_run_sql( @@ -1448,7 +1497,10 @@ mod client_query_tests { ), ) .await; - assert!(rejected.is_err(), "preset over a missing column should be rejected"); + assert!( + rejected.is_err(), + "preset over a missing column should be rejected" + ); // DROP removes it; the document becomes empty. run_sql(&runtime, &format!("DROP EXTENSION 'preset' FOR {table}")).await; @@ -1466,7 +1518,10 @@ mod client_query_tests { /// assert on failures from side-effecting statements. async fn try_run_sql(runtime: &Runtime, sql: &str) -> anyhow::Result<()> { runtime - .run_query(crate::query::Query::sql(sql.to_string()), beacon_auth::AuthIdentity::system()) + .run_query( + crate::query::Query::sql(sql.to_string()), + beacon_auth::AuthIdentity::system(), + ) .await? .into_record_stream()? .try_collect::>() @@ -1478,7 +1533,11 @@ mod client_query_tests { /// file download. #[tokio::test(flavor = "multi_thread")] async fn query_with_output_format_produces_file() { - let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new(beacon_config::Config::load().unwrap())).await.expect("runtime should start"); + let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new( + beacon_config::Config::load().unwrap(), + )) + .await + .expect("runtime should start"); let suffix = uuid::Uuid::new_v4().simple(); let table = format!("out_{suffix}"); @@ -1517,7 +1576,9 @@ mod client_query_tests { async fn query_with_netcdf_output_writes_under_configured_tmp() { let config = std::sync::Arc::new(beacon_config::Config::load().unwrap()); let tmp_dir = config.storage.tmp_dir.clone(); - let runtime = Runtime::new_with_in_memory_auth(config).await.expect("runtime should start"); + let runtime = Runtime::new_with_in_memory_auth(config) + .await + .expect("runtime should start"); let suffix = uuid::Uuid::new_v4().simple(); let table = format!("ncout_{suffix}"); @@ -1561,7 +1622,11 @@ mod client_query_tests { /// operation (super-user-only). #[tokio::test(flavor = "multi_thread")] async fn non_super_user_is_gated_by_validation() { - let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new(beacon_config::Config::load().unwrap())).await.expect("runtime should start"); + let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new( + beacon_config::Config::load().unwrap(), + )) + .await + .expect("runtime should start"); let suffix = uuid::Uuid::new_v4().simple(); let table = format!("val_{suffix}"); @@ -1570,7 +1635,10 @@ mod client_query_tests { // Non-super-user: read-only SELECT is allowed. runtime - .run_query(crate::query::Query::sql(format!("SELECT * FROM {table}")), beacon_auth::AuthIdentity::empty()) + .run_query( + crate::query::Query::sql(format!("SELECT * FROM {table}")), + beacon_auth::AuthIdentity::empty(), + ) .await .expect("non-super SELECT should be allowed") .into_record_stream() @@ -1667,14 +1735,24 @@ mod client_query_tests { /// planner, execs, and Lance index ops through the full SQL path. #[tokio::test(flavor = "multi_thread")] async fn create_show_drop_index_round_trip() { - let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new(beacon_config::Config::load().unwrap())) - .await - .expect("runtime should start"); + let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new( + beacon_config::Config::load().unwrap(), + )) + .await + .expect("runtime should start"); let suffix = uuid::Uuid::new_v4().simple(); let table = format!("idx_{suffix}"); - run_sql(&runtime, &format!("CREATE TABLE {table} (id BIGINT, name VARCHAR)")).await; - run_sql(&runtime, &format!("INSERT INTO {table} VALUES (1, 'a'), (2, 'b')")).await; + run_sql( + &runtime, + &format!("CREATE TABLE {table} (id BIGINT, name VARCHAR)"), + ) + .await; + run_sql( + &runtime, + &format!("INSERT INTO {table} VALUES (1, 'a'), (2, 'b')"), + ) + .await; run_sql( &runtime, &format!("CREATE INDEX {table}_id_idx ON {table} (id) USING btree"), @@ -1685,7 +1763,10 @@ mod client_query_tests { let runtime = &runtime; async move { runtime - .run_query(crate::query::Query::sql(sql), beacon_auth::AuthIdentity::system()) + .run_query( + crate::query::Query::sql(sql), + beacon_auth::AuthIdentity::system(), + ) .await .expect("show indexes should run") .into_record_stream() @@ -1723,7 +1804,10 @@ mod restart_tests { async fn run_sql(runtime: &Runtime, sql: &str) { runtime - .run_query(crate::query::Query::sql(sql.to_string()), beacon_auth::AuthIdentity::system()) + .run_query( + crate::query::Query::sql(sql.to_string()), + beacon_auth::AuthIdentity::system(), + ) .await .expect("sql should run") .into_record_stream() @@ -1735,7 +1819,10 @@ mod restart_tests { async fn count_rows(runtime: &Runtime, sql: &str) -> usize { runtime - .run_query(crate::query::Query::sql(sql.to_string()), beacon_auth::AuthIdentity::system()) + .run_query( + crate::query::Query::sql(sql.to_string()), + beacon_auth::AuthIdentity::system(), + ) .await .expect("sql should run") .into_record_stream() @@ -1764,15 +1851,23 @@ mod restart_tests { let config = std::sync::Arc::new(beacon_config::Config::load().unwrap()); // First runtime: create a base table with data and a view over it. - let runtime = Runtime::new_with_in_memory_auth(config.clone()).await.expect("runtime should start"); + let runtime = Runtime::new_with_in_memory_auth(config.clone()) + .await + .expect("runtime should start"); run_sql(&runtime, &format!("CREATE TABLE {base} (a BIGINT)")).await; run_sql(&runtime, &format!("INSERT INTO {base} VALUES (1), (2)")).await; - run_sql(&runtime, &format!("CREATE VIEW {view} AS SELECT a FROM {base}")).await; + run_sql( + &runtime, + &format!("CREATE VIEW {view} AS SELECT a FROM {base}"), + ) + .await; drop(runtime); // A fresh runtime rebuilds the catalog purely from the persisted // `tables:///table.json` definitions. - let restarted = Runtime::new_with_in_memory_auth(config).await.expect("runtime should restart"); + let restarted = Runtime::new_with_in_memory_auth(config) + .await + .expect("runtime should restart"); assert_eq!( count_rows(&restarted, &format!("SELECT * FROM {base}")).await, @@ -1788,10 +1883,16 @@ mod restart_tests { // Cleanup so the shared on-disk tables store does not leak into other // tests (best-effort; `DROP TABLE` deregisters either provider type). let _ = restarted - .run_query(crate::query::Query::sql(format!("DROP TABLE {view}")), beacon_auth::AuthIdentity::system()) + .run_query( + crate::query::Query::sql(format!("DROP TABLE {view}")), + beacon_auth::AuthIdentity::system(), + ) .await; let _ = restarted - .run_query(crate::query::Query::sql(format!("DROP TABLE {base}")), beacon_auth::AuthIdentity::system()) + .run_query( + crate::query::Query::sql(format!("DROP TABLE {base}")), + beacon_auth::AuthIdentity::system(), + ) .await; } } @@ -1854,9 +1955,7 @@ mod external_table_sql_tests { #[test] fn rejects_unsafe_identifiers() { assert!(build_create_external_table_sql(&req("bad name", "PARQUET", "x/")).is_err()); - assert!( - build_create_external_table_sql(&req("t; DROP TABLE u", "PARQUET", "x/")).is_err() - ); + assert!(build_create_external_table_sql(&req("t; DROP TABLE u", "PARQUET", "x/")).is_err()); assert!(build_create_external_table_sql(&req("t", "PARQUET'", "x/")).is_err()); let mut r = req("t", "PARQUET", "x/"); @@ -1876,9 +1975,11 @@ mod crawler_admin_tests { /// second drop). Uses a unique name so the shared on-disk store does not leak. #[tokio::test(flavor = "multi_thread")] async fn crawler_create_list_get_run_drop_round_trip() { - let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new(beacon_config::Config::load().unwrap())) - .await - .expect("runtime should start"); + let runtime = Runtime::new_with_in_memory_auth(std::sync::Arc::new( + beacon_config::Config::load().unwrap(), + )) + .await + .expect("runtime should start"); let name = format!("crawler_test_{}", uuid::Uuid::new_v4().simple()); diff --git a/beacon-core/src/statement_plan/actions.rs b/beacon-core/src/statement_plan/actions.rs index f7ea7e55..a5174ccf 100644 --- a/beacon-core/src/statement_plan/actions.rs +++ b/beacon-core/src/statement_plan/actions.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use beacon_data_lake::DATASETS_OBJECT_STORE_URL; use beacon_datafusion_ext::{ listing_table_factory_ext::ListingTableFactoryExt, - remote::{RemoteTableDefinition, unresolved_schema}, + remote::{unresolved_schema, RemoteTableDefinition}, table_ext::{MaterializedView, TableDefinition}, }; use beacon_sql_databases::{EncryptedSecret, SqlDatabaseTableDefinition, SqlEngine}; @@ -181,8 +181,14 @@ fn parse_remote_location( ) })?; - anyhow::ensure!(!authority.is_empty(), "remote table LOCATION missing host:port"); - anyhow::ensure!(!table.is_empty(), "remote table LOCATION missing table name"); + anyhow::ensure!( + !authority.is_empty(), + "remote table LOCATION missing host:port" + ); + anyhow::ensure!( + !table.is_empty(), + "remote table LOCATION missing table name" + ); let tls = tls_option .map(|v| v.eq_ignore_ascii_case("true")) @@ -525,11 +531,15 @@ pub(crate) async fn alter_table( }); } other => { - return Err(anyhow::anyhow!("Unsupported ALTER COLUMN operation: {other}")); + return Err(anyhow::anyhow!( + "Unsupported ALTER COLUMN operation: {other}" + )); } }, other => { - return Err(anyhow::anyhow!("Unsupported ALTER TABLE operation: {other}")); + return Err(anyhow::anyhow!( + "Unsupported ALTER TABLE operation: {other}" + )); } } } diff --git a/beacon-core/src/statement_plan/auth.rs b/beacon-core/src/statement_plan/auth.rs index bfd9b210..ae7376fa 100644 --- a/beacon-core/src/statement_plan/auth.rs +++ b/beacon-core/src/statement_plan/auth.rs @@ -39,14 +39,21 @@ pub(crate) fn apply_auth_statement( AuthStatement::RevokeRoleFromUser { role, username } => { auth.revoke_role_from_user(username, role) } - AuthStatement::GrantPrivilege { privilege, target, role } => { - auth.grant(role, PrivilegeRule::new(*privilege, target.clone())) - } - AuthStatement::DenyPrivilege { privilege, target, role } => { - auth.deny(role, PrivilegeRule::new(*privilege, target.clone())) - } - AuthStatement::RevokePrivilege { privilege, target, role, deny } => { - auth.revoke(role, &PrivilegeRule::new(*privilege, target.clone()), *deny) - } + AuthStatement::GrantPrivilege { + privilege, + target, + role, + } => auth.grant(role, PrivilegeRule::new(*privilege, target.clone())), + AuthStatement::DenyPrivilege { + privilege, + target, + role, + } => auth.deny(role, PrivilegeRule::new(*privilege, target.clone())), + AuthStatement::RevokePrivilege { + privilege, + target, + role, + deny, + } => auth.revoke(role, &PrivilegeRule::new(*privilege, target.clone()), *deny), } } diff --git a/beacon-core/src/statement_plan/authz.rs b/beacon-core/src/statement_plan/authz.rs index 7026d503..89a4eece 100644 --- a/beacon-core/src/statement_plan/authz.rs +++ b/beacon-core/src/statement_plan/authz.rs @@ -154,8 +154,8 @@ mod tests { let ctx = SessionContext::new_with_config(SessionConfig::new().with_information_schema(true)); let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); - let batch = - RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1]))]).unwrap(); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1]))]) + .unwrap(); let table = MemTable::try_new(schema, vec![vec![batch]]).unwrap(); ctx.register_table(name, Arc::new(table)).unwrap(); ctx @@ -180,10 +180,14 @@ mod tests { let plan = plan_for(&ctx, "SELECT * FROM observations").await; let denied = auth_with_reader_grant(None); - assert!(authorize_logical_plan(&plan, &ctx, &denied, &identity(&["reader"]), true).is_err()); + assert!( + authorize_logical_plan(&plan, &ctx, &denied, &identity(&["reader"]), true).is_err() + ); let allowed = auth_with_reader_grant(Some(PrivilegeRule::new(Privilege::Select, None))); - assert!(authorize_logical_plan(&plan, &ctx, &allowed, &identity(&["reader"]), true).is_ok()); + assert!( + authorize_logical_plan(&plan, &ctx, &allowed, &identity(&["reader"]), true).is_ok() + ); } #[tokio::test] diff --git a/beacon-core/src/statement_plan/crawler.rs b/beacon-core/src/statement_plan/crawler.rs index 56231cd3..44617a70 100644 --- a/beacon-core/src/statement_plan/crawler.rs +++ b/beacon-core/src/statement_plan/crawler.rs @@ -75,8 +75,12 @@ pub(crate) async fn show_crawlers(session: &Arc) -> anyhow::Resu .iter() .map(|c| c.format_filter.as_ref().map(|f| f.join(","))) .collect(); - let detect_partitions = - BooleanArray::from(crawlers.iter().map(|c| c.detect_partitions).collect::>()); + let detect_partitions = BooleanArray::from( + crawlers + .iter() + .map(|c| c.detect_partitions) + .collect::>(), + ); let schedule_secs = UInt64Array::from(crawlers.iter().map(|c| c.schedule_secs).collect::>()); let event_driven = diff --git a/beacon-core/src/statement_plan/logical.rs b/beacon-core/src/statement_plan/logical.rs index ebe17592..f61d4797 100644 --- a/beacon-core/src/statement_plan/logical.rs +++ b/beacon-core/src/statement_plan/logical.rs @@ -126,11 +126,7 @@ impl UserDefinedLogicalNodeCore for CreateMaterializedViewNode { write!(f, "CreateMaterializedView: name={}", self.view_name) } - fn with_exprs_and_inputs( - &self, - _exprs: Vec, - _inputs: Vec, - ) -> Result { + fn with_exprs_and_inputs(&self, _exprs: Vec, _inputs: Vec) -> Result { Ok(Self { view_name: self.view_name.clone(), query_sql: self.query_sql.clone(), @@ -203,11 +199,7 @@ impl UserDefinedLogicalNodeCore for RefreshNode { write!(f, "Refresh: name={}", self.name) } - fn with_exprs_and_inputs( - &self, - _exprs: Vec, - _inputs: Vec, - ) -> Result { + fn with_exprs_and_inputs(&self, _exprs: Vec, _inputs: Vec) -> Result { Ok(Self { name: self.name.clone(), }) @@ -434,7 +426,11 @@ impl UserDefinedLogicalNodeCore for CreateIndexNode { vec![] } fn fmt_for_explain(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "CreateIndex: table={} column={}", self.table, self.column) + write!( + f, + "CreateIndex: table={} column={}", + self.table, self.column + ) } fn with_exprs_and_inputs(&self, _exprs: Vec, _inputs: Vec) -> Result { Ok(Self { @@ -551,7 +547,11 @@ impl UserDefinedLogicalNodeCore for ReplaceTableContentsNode { fn fmt_for_explain(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "ReplaceTableContents: table={}", self.table) } - fn with_exprs_and_inputs(&self, _exprs: Vec, mut inputs: Vec) -> Result { + fn with_exprs_and_inputs( + &self, + _exprs: Vec, + mut inputs: Vec, + ) -> Result { Ok(Self { table: self.table.clone(), input: inputs.swap_remove(0), diff --git a/beacon-core/src/statement_plan/lower.rs b/beacon-core/src/statement_plan/lower.rs index 5d6a0bc7..72a8a143 100644 --- a/beacon-core/src/statement_plan/lower.rs +++ b/beacon-core/src/statement_plan/lower.rs @@ -72,7 +72,10 @@ pub(crate) async fn lower_df_statement( // DataFusion has no `ALTER TABLE` planning, so build the node from the AST. if let datafusion::sql::parser::Statement::Statement(sql_stmt) = &statement { if let SqlAstStatement::AlterTable(alter) = sql_stmt.as_ref() { - return Ok(alter_table_plan(alter.name.clone(), alter.operations.clone())); + return Ok(alter_table_plan( + alter.name.clone(), + alter.operations.clone(), + )); } } @@ -182,7 +185,11 @@ fn update_plan(dml: DmlStatement) -> anyhow::Result { }; let mutation = update_mutation(projection); - Ok(replace_contents_plan(dml.table_name, new_contents, mutation)) + Ok(replace_contents_plan( + dml.table_name, + new_contents, + mutation, + )) } /// Derive a native `UPDATE` spec from the planned projection (best-effort): diff --git a/beacon-core/src/statement_plan/mod.rs b/beacon-core/src/statement_plan/mod.rs index 424d9e86..aa23141e 100644 --- a/beacon-core/src/statement_plan/mod.rs +++ b/beacon-core/src/statement_plan/mod.rs @@ -221,7 +221,9 @@ pub(crate) fn drop_extension_plan(statement: DropExtensionStatement) -> LogicalP /// Build the logical plan for `SHOW EXTENSIONS FOR
`. pub(crate) fn show_extensions_plan(statement: ShowExtensionsStatement) -> LogicalPlan { LogicalPlan::Extension(Extension { - node: Arc::new(logical::ShowExtensionsNode::new(statement.table.to_string())), + node: Arc::new(logical::ShowExtensionsNode::new( + statement.table.to_string(), + )), }) } diff --git a/beacon-core/src/statement_plan/physical.rs b/beacon-core/src/statement_plan/physical.rs index ede31dbe..d4a912d3 100644 --- a/beacon-core/src/statement_plan/physical.rs +++ b/beacon-core/src/statement_plan/physical.rs @@ -799,7 +799,9 @@ side_effect_exec!(RunCrawlerExec, "RunCrawlerExec", |exec: &RunCrawlerExec| { let session = upgrade_session(&exec.session)?; let name = exec.name.clone(); Ok(side_effect_stream(async move { - crawler::run_crawler(&session, &name).await.map_err(to_df_err) + crawler::run_crawler(&session, &name) + .await + .map_err(to_df_err) })) }); @@ -858,7 +860,9 @@ impl ShowCrawlersExec { impl DisplayAs for ShowCrawlersExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match t { - DisplayFormatType::Default | DisplayFormatType::Verbose => write!(f, "ShowCrawlersExec"), + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "ShowCrawlersExec") + } DisplayFormatType::TreeRender => write!(f, "ShowCrawlersExec"), } } @@ -926,22 +930,30 @@ impl CreateIndexExec { } } fn fmt_label(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "CreateIndexExec: table={} column={}", self.table, self.column) + write!( + f, + "CreateIndexExec: table={} column={}", + self.table, self.column + ) } } -side_effect_exec!(CreateIndexExec, "CreateIndexExec", |exec: &CreateIndexExec| { - let session = upgrade_session(&exec.session)?; - let table = exec.table.clone(); - let column = exec.column.clone(); - let name = exec.name.clone(); - let using = exec.using.clone(); - Ok(side_effect_stream(async move { - actions::create_index(&session, &table, &column, name, using) - .await - .map_err(to_df_err) - })) -}); +side_effect_exec!( + CreateIndexExec, + "CreateIndexExec", + |exec: &CreateIndexExec| { + let session = upgrade_session(&exec.session)?; + let table = exec.table.clone(); + let column = exec.column.clone(); + let name = exec.name.clone(); + let using = exec.using.clone(); + Ok(side_effect_stream(async move { + actions::create_index(&session, &table, &column, name, using) + .await + .map_err(to_df_err) + })) + } +); /// Physical node for `DROP INDEX ON
`. #[derive(Debug)] @@ -1034,7 +1046,9 @@ impl ExecutionPlan for ShowIndexesExec { let table = self.table.clone(); let schema = show_indexes_arrow_schema(); let stream = futures::stream::once(async move { - actions::list_indexes(&session, &table).await.map_err(to_df_err) + actions::list_indexes(&session, &table) + .await + .map_err(to_df_err) }); Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) } @@ -1061,7 +1075,11 @@ impl SetExtensionExec { } } fn fmt_label(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "SetExtensionExec: table={} kind={}", self.table, self.kind) + write!( + f, + "SetExtensionExec: table={} kind={}", + self.table, self.kind + ) } } @@ -1100,7 +1118,11 @@ impl DropExtensionExec { } } fn fmt_label(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "DropExtensionExec: table={} kind={}", self.table, self.kind) + write!( + f, + "DropExtensionExec: table={} kind={}", + self.table, self.kind + ) } } diff --git a/beacon-core/tests/auth.rs b/beacon-core/tests/auth.rs index 990f342c..68dda344 100644 --- a/beacon-core/tests/auth.rs +++ b/beacon-core/tests/auth.rs @@ -80,7 +80,10 @@ async fn admin_credentials_authenticate_as_super_user() { let (runtime, config) = runtime_with(config(false, true)).await; let identity = admin_identity(&runtime, &config).await; assert_eq!(identity.username, config.admin.username); - assert!(identity.is_super_user, "the bootstrapped admin is a super-user"); + assert!( + identity.is_super_user, + "the bootstrapped admin is a super-user" + ); } #[tokio::test(flavor = "multi_thread")] @@ -242,7 +245,11 @@ async fn seed_two_tables_and_reader(runtime: &Runtime) -> (String, String, AuthI exec_admin(runtime, &format!("CREATE ROLE {role}")).await; exec_admin(runtime, &format!("CREATE USER {user} WITH PASSWORD 'pw'")).await; exec_admin(runtime, &format!("GRANT ROLE {role} TO USER {user}")).await; - exec_admin(runtime, &format!("GRANT SELECT ON TABLE {t1} TO ROLE {role}")).await; + exec_admin( + runtime, + &format!("GRANT SELECT ON TABLE {t1} TO ROLE {role}"), + ) + .await; let identity = runtime .authenticate(&Credential::basic(user, "pw")) @@ -283,16 +290,22 @@ async fn enforced_deny_wins_over_grant() { exec_admin(&runtime, &format!("GRANT ROLE {role} TO USER {user}")).await; // Grant SELECT on everything, then deny one table — deny must win. exec_admin(&runtime, &format!("GRANT SELECT TO ROLE {role}")).await; - exec_admin(&runtime, &format!("DENY SELECT ON TABLE {t2} TO ROLE {role}")).await; + exec_admin( + &runtime, + &format!("DENY SELECT ON TABLE {t2} TO ROLE {role}"), + ) + .await; let alice = runtime .authenticate(&Credential::basic(user, "pw")) .await .unwrap(); - assert!(exec(&runtime, &format!("SELECT * FROM {t1}"), alice.clone()) - .await - .is_ok()); + assert!( + exec(&runtime, &format!("SELECT * FROM {t1}"), alice.clone()) + .await + .is_ok() + ); assert!( exec(&runtime, &format!("SELECT * FROM {t2}"), alice) .await @@ -316,7 +329,11 @@ async fn enforced_revoke_removes_access() { exec_admin(&runtime, &format!("CREATE ROLE {role}")).await; exec_admin(&runtime, &format!("CREATE USER {user} WITH PASSWORD 'pw'")).await; exec_admin(&runtime, &format!("GRANT ROLE {role} TO USER {user}")).await; - exec_admin(&runtime, &format!("GRANT SELECT ON TABLE {table} TO ROLE {role}")).await; + exec_admin( + &runtime, + &format!("GRANT SELECT ON TABLE {table} TO ROLE {role}"), + ) + .await; let u = runtime .authenticate(&Credential::basic(user, "pw")) .await @@ -326,7 +343,11 @@ async fn enforced_revoke_removes_access() { .await .is_ok()); - exec_admin(&runtime, &format!("REVOKE SELECT ON TABLE {table} FROM ROLE {role}")).await; + exec_admin( + &runtime, + &format!("REVOKE SELECT ON TABLE {table} FROM ROLE {role}"), + ) + .await; assert!( exec(&runtime, &format!("SELECT * FROM {table}"), u) .await @@ -375,8 +396,16 @@ async fn enforcement_off_allows_ungranted_reads() { exec_admin(&runtime, &format!("INSERT INTO {table} VALUES (1)")).await; // A role-less identity can still read when enforcement is off (backwards-compatible default). - let result = exec(&runtime, &format!("SELECT * FROM {table}"), AuthIdentity::empty()).await; - assert_eq!(total_rows(&result.expect("read allowed when enforce=off")), 1); + let result = exec( + &runtime, + &format!("SELECT * FROM {table}"), + AuthIdentity::empty(), + ) + .await; + assert_eq!( + total_rows(&result.expect("read allowed when enforce=off")), + 1 + ); } #[tokio::test(flavor = "multi_thread")] @@ -392,7 +421,10 @@ async fn information_schema_is_exempt_from_enforcement() { AuthIdentity::empty(), ) .await; - assert!(result.is_ok(), "information_schema must be readable: {result:?}"); + assert!( + result.is_ok(), + "information_schema must be readable: {result:?}" + ); } // -------------------------------------------------------------------------------------------- @@ -428,9 +460,13 @@ async fn non_super_user_cannot_manage_auth_or_run_ddl() { } // Standard DDL is also rejected for a non-super-user. - assert!(exec(&runtime, &format!("CREATE TABLE {} (a BIGINT)", unique("x")), alice) - .await - .is_err()); + assert!(exec( + &runtime, + &format!("CREATE TABLE {} (a BIGINT)", unique("x")), + alice + ) + .await + .is_err()); } // -------------------------------------------------------------------------------------------- @@ -491,7 +527,10 @@ async fn enforced_path_grant_matches_only_granted_prefix() { alice.clone(), ) .await; - assert!(allowed.is_ok(), "granted path should be readable: {allowed:?}"); + assert!( + allowed.is_ok(), + "granted path should be readable: {allowed:?}" + ); let denied = exec( &runtime, diff --git a/beacon-core/tests/crawler_csv.rs b/beacon-core/tests/crawler_csv.rs index 557ff56e..116171ef 100644 --- a/beacon-core/tests/crawler_csv.rs +++ b/beacon-core/tests/crawler_csv.rs @@ -14,7 +14,10 @@ use futures::TryStreamExt; async fn run(runtime: &Runtime, sql: &str) -> Vec { runtime - .run_query(Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await .unwrap_or_else(|e| panic!("SQL failed: {sql}\n{e}")) .into_record_stream() @@ -62,7 +65,10 @@ async fn crawler_discovers_partitioned_csv() { let pruned = scalar_count(&run(&runtime, "SELECT count(*) FROM csv_src WHERE year = '2024'").await); - assert_eq!(pruned, 1, "partition column 'year' should filter to one file"); + assert_eq!( + pruned, 1, + "partition column 'year' should filter to one file" + ); let _ = std::fs::remove_dir_all(&tmp); } diff --git a/beacon-core/tests/crawler_e2e.rs b/beacon-core/tests/crawler_e2e.rs index a0580bca..89017461 100644 --- a/beacon-core/tests/crawler_e2e.rs +++ b/beacon-core/tests/crawler_e2e.rs @@ -16,7 +16,10 @@ use futures::TryStreamExt; /// Run SQL as super-user and collect the result batches. async fn run(runtime: &Runtime, sql: &str) -> Vec { runtime - .run_query(Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await .unwrap_or_else(|error| panic!("SQL failed to plan/execute: {sql}\n{error}")) .into_record_stream() @@ -92,9 +95,17 @@ async fn crawler_discovers_partitioned_parquet() { assert_eq!(count, 2, "both partition files should be discovered"); // The Hive partition column 'year' exists and prunes correctly. - let pruned = - scalar_count(&run(&runtime, "SELECT count(*) FROM crawl_src WHERE year = '2024'").await); - assert_eq!(pruned, 1, "partition column 'year' should filter to one file"); + let pruned = scalar_count( + &run( + &runtime, + "SELECT count(*) FROM crawl_src WHERE year = '2024'", + ) + .await, + ); + assert_eq!( + pruned, 1, + "partition column 'year' should filter to one file" + ); // Re-running is idempotent: the crawler owns the table, so it updates (not // skips) it, and the row count is unchanged. diff --git a/beacon-core/tests/crawler_events.rs b/beacon-core/tests/crawler_events.rs index 60525541..9da663a7 100644 --- a/beacon-core/tests/crawler_events.rs +++ b/beacon-core/tests/crawler_events.rs @@ -19,7 +19,10 @@ use futures::TryStreamExt; async fn run(runtime: &Runtime, sql: &str) -> Vec { runtime - .run_query(Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await .unwrap_or_else(|e| panic!("SQL failed: {sql}\n{e}")) .into_record_stream() @@ -31,7 +34,10 @@ async fn run(runtime: &Runtime, sql: &str) -> Vec { async fn try_count(runtime: &Runtime, table: &str) -> Option { let result = runtime - .run_query(Query::sql(format!("SELECT count(*) FROM {table}")), beacon_core::AuthIdentity::system()) + .run_query( + Query::sql(format!("SELECT count(*) FROM {table}")), + beacon_core::AuthIdentity::system(), + ) .await .ok()?; let batches: Vec = result.into_record_stream().ok()?.try_collect().await.ok()?; diff --git a/beacon-core/tests/crawler_scheduled.rs b/beacon-core/tests/crawler_scheduled.rs index 513a9e72..42a04083 100644 --- a/beacon-core/tests/crawler_scheduled.rs +++ b/beacon-core/tests/crawler_scheduled.rs @@ -18,7 +18,10 @@ use futures::TryStreamExt; async fn run(runtime: &Runtime, sql: &str) -> Vec { runtime - .run_query(Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await .unwrap_or_else(|e| panic!("SQL failed: {sql}\n{e}")) .into_record_stream() @@ -82,7 +85,11 @@ async fn scheduled_crawler_runs_without_manual_trigger() { ])); // Define a crawler with a 1s schedule but DO NOT run it manually. - run(&runtime, "CREATE CRAWLER sched ON 'sched_src/' WITH ('schedule' '1s')").await; + run( + &runtime, + "CREATE CRAWLER sched ON 'sched_src/' WITH ('schedule' '1s')", + ) + .await; // Now drop data in — a subsequent scheduled tick must discover and register it. write_parquet(&datasets.join("sched_src/a.parquet"), &schema); diff --git a/beacon-core/tests/crawler_zarr.rs b/beacon-core/tests/crawler_zarr.rs index cad70cf9..bd1e0b0a 100644 --- a/beacon-core/tests/crawler_zarr.rs +++ b/beacon-core/tests/crawler_zarr.rs @@ -20,7 +20,10 @@ use futures::TryStreamExt; async fn run(runtime: &Runtime, sql: &str) -> Vec { runtime - .run_query(Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await .unwrap_or_else(|e| panic!("SQL failed: {sql}\n{e}")) .into_record_stream() @@ -33,7 +36,10 @@ async fn run(runtime: &Runtime, sql: &str) -> Vec { /// `SELECT count(*)`; `None` when the table does not exist. async fn try_count(runtime: &Runtime, table: &str) -> Option { let result = runtime - .run_query(Query::sql(format!("SELECT count(*) FROM {table}")), beacon_core::AuthIdentity::system()) + .run_query( + Query::sql(format!("SELECT count(*) FROM {table}")), + beacon_core::AuthIdentity::system(), + ) .await .ok()?; let batches: Vec = result.into_record_stream().ok()?.try_collect().await.ok()?; diff --git a/beacon-core/tests/delta_tables.rs b/beacon-core/tests/delta_tables.rs index a36a9924..953fa9dc 100644 --- a/beacon-core/tests/delta_tables.rs +++ b/beacon-core/tests/delta_tables.rs @@ -14,7 +14,10 @@ use futures::TryStreamExt; /// Run SQL as a super-user and collect the result batches. async fn run(runtime: &Runtime, sql: &str) -> Vec { runtime - .run_query(beacon_core::query::Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + beacon_core::query::Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await .unwrap_or_else(|error| panic!("SQL failed to plan/execute: {sql}\n{error}")) .into_record_stream() @@ -98,7 +101,10 @@ async fn delta_external_table_read_insert_time_travel_drop() { ) .await, ); - assert_eq!(v0_count, 2, "read_delta(.., 0) should see version 0 (2 rows)"); + assert_eq!( + v0_count, 2, + "read_delta(.., 0) should see version 0 (2 rows)" + ); // CREATE EXTERNAL TABLE ... STORED AS DELTA, then SELECT. run( @@ -111,8 +117,13 @@ async fn delta_external_table_read_insert_time_travel_drop() { // INSERT INTO appends a new Delta version. run(&runtime, &format!("INSERT INTO {table} VALUES (5), (6)")).await; - let after_insert = - scalar_count(&run(&runtime, &format!("SELECT count(*) FROM read_delta('{location}')")).await); + let after_insert = scalar_count( + &run( + &runtime, + &format!("SELECT count(*) FROM read_delta('{location}')"), + ) + .await, + ); assert_eq!(after_insert, 6, "INSERT INTO should commit two more rows"); // DROP TABLE deregisters it (the underlying Delta files remain on disk). diff --git a/beacon-core/tests/explain_analyze_external_netcdf.rs b/beacon-core/tests/explain_analyze_external_netcdf.rs index 4df27ef7..bda13496 100644 --- a/beacon-core/tests/explain_analyze_external_netcdf.rs +++ b/beacon-core/tests/explain_analyze_external_netcdf.rs @@ -18,7 +18,10 @@ use futures::TryStreamExt; async fn collect(runtime: &Runtime, sql: &str) -> anyhow::Result> { Ok(runtime - .run_query(beacon_core::query::Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + beacon_core::query::Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await? .into_record_stream()? .try_collect::>() diff --git a/beacon-core/tests/explain_analyze_pg_json.rs b/beacon-core/tests/explain_analyze_pg_json.rs index 8cc249c1..917d244b 100644 --- a/beacon-core/tests/explain_analyze_pg_json.rs +++ b/beacon-core/tests/explain_analyze_pg_json.rs @@ -9,7 +9,10 @@ use futures::TryStreamExt; async fn run_sql(runtime: &Runtime, sql: &str) { runtime - .run_query(Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await .expect("sql should run") .into_record_stream() @@ -35,10 +38,17 @@ async fn explain_analyze_returns_pg_json_with_metrics() { let suffix = uuid::Uuid::new_v4().simple(); let table = format!("explain_analyze_pgjson_{suffix}"); run_sql(&runtime, &format!("CREATE TABLE {table} (a BIGINT)")).await; - run_sql(&runtime, &format!("INSERT INTO {table} VALUES (1), (2), (3)")).await; + run_sql( + &runtime, + &format!("INSERT INTO {table} VALUES (1), (2), (3)"), + ) + .await; let json_str = runtime - .explain_analyze_client_query(sql_query(&format!("SELECT a FROM {table}")), beacon_core::AuthIdentity::empty()) + .explain_analyze_client_query( + sql_query(&format!("SELECT a FROM {table}")), + beacon_core::AuthIdentity::empty(), + ) .await .expect("explain analyze should succeed"); diff --git a/beacon-core/tests/iceberg_tables.rs b/beacon-core/tests/iceberg_tables.rs index 9d8b1f21..7ea6c98e 100644 --- a/beacon-core/tests/iceberg_tables.rs +++ b/beacon-core/tests/iceberg_tables.rs @@ -9,7 +9,10 @@ use futures::TryStreamExt; /// Run SQL as a super-user (DDL/DML allowed) and collect the result batches. async fn run(runtime: &Runtime, sql: &str) -> Vec { runtime - .run_query(beacon_core::query::Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + beacon_core::query::Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await .unwrap_or_else(|error| panic!("SQL failed to plan/execute: {sql}\n{error}")) .into_record_stream() @@ -42,7 +45,9 @@ fn scalar_string(batches: &[RecordBatch]) -> String { #[tokio::test(flavor = "multi_thread")] async fn iceberg_create_insert_select_ctas_drop() { - let runtime = Runtime::new(std::sync::Arc::new(beacon_config::Config::load().unwrap())).await.expect("runtime should boot"); + let runtime = Runtime::new(std::sync::Arc::new(beacon_config::Config::load().unwrap())) + .await + .expect("runtime should boot"); // The default managed-table engine is Lance; this test exercises the Iceberg // path, so select it explicitly for this session. @@ -52,12 +57,30 @@ async fn iceberg_create_insert_select_ctas_drop() { // warehouse, and clean any leftovers from a previous aborted run. let table = format!("ice_e2e_{}", std::process::id()); let copy = format!("{table}_copy"); - let _ = runtime.run_query(beacon_core::query::Query::sql(format!("DROP TABLE IF EXISTS {table}")), beacon_core::AuthIdentity::system()).await; - let _ = runtime.run_query(beacon_core::query::Query::sql(format!("DROP TABLE IF EXISTS {copy}")), beacon_core::AuthIdentity::system()).await; + let _ = runtime + .run_query( + beacon_core::query::Query::sql(format!("DROP TABLE IF EXISTS {table}")), + beacon_core::AuthIdentity::system(), + ) + .await; + let _ = runtime + .run_query( + beacon_core::query::Query::sql(format!("DROP TABLE IF EXISTS {copy}")), + beacon_core::AuthIdentity::system(), + ) + .await; // CREATE + INSERT + SELECT. - run(&runtime, &format!("CREATE TABLE {table} (id BIGINT, name VARCHAR)")).await; - run(&runtime, &format!("INSERT INTO {table} VALUES (1, 'a'), (2, 'b')")).await; + run( + &runtime, + &format!("CREATE TABLE {table} (id BIGINT, name VARCHAR)"), + ) + .await; + run( + &runtime, + &format!("INSERT INTO {table} VALUES (1, 'a'), (2, 'b')"), + ) + .await; let count = scalar_count(&run(&runtime, &format!("SELECT count(*) FROM {table}")).await); assert_eq!(count, 2, "two inserted rows should be visible"); @@ -75,42 +98,74 @@ async fn iceberg_create_insert_select_ctas_drop() { ); // CREATE TABLE AS SELECT. - run(&runtime, &format!("CREATE TABLE {copy} AS SELECT * FROM {table}")).await; + run( + &runtime, + &format!("CREATE TABLE {copy} AS SELECT * FROM {table}"), + ) + .await; let copy_count = scalar_count(&run(&runtime, &format!("SELECT count(*) FROM {copy}")).await); assert_eq!(copy_count, 2, "CTAS should copy all rows"); // UPDATE WHERE: only the matching row changes; the other is untouched. - run(&runtime, &format!("UPDATE {table} SET name = 'Z' WHERE id = 1")).await; - let updated = scalar_string(&run(&runtime, &format!("SELECT name FROM {table} WHERE id = 1")).await); + run( + &runtime, + &format!("UPDATE {table} SET name = 'Z' WHERE id = 1"), + ) + .await; + let updated = + scalar_string(&run(&runtime, &format!("SELECT name FROM {table} WHERE id = 1")).await); assert_eq!(updated, "Z", "UPDATE WHERE id = 1 should set name to 'Z'"); - let untouched = scalar_string(&run(&runtime, &format!("SELECT name FROM {table} WHERE id = 2")).await); + let untouched = + scalar_string(&run(&runtime, &format!("SELECT name FROM {table} WHERE id = 2")).await); assert_eq!(untouched, "b", "non-matching row should be unchanged"); let row_count = scalar_count(&run(&runtime, &format!("SELECT count(*) FROM {table}")).await); assert_eq!(row_count, 2, "UPDATE must not change the row count"); // UPDATE all rows (no WHERE). run(&runtime, &format!("UPDATE {table} SET name = 'all'")).await; - let distinct_names = - scalar_count(&run(&runtime, &format!("SELECT count(DISTINCT name) FROM {table}")).await); - assert_eq!(distinct_names, 1, "UPDATE without WHERE should set every row"); - let any_name = scalar_string(&run(&runtime, &format!("SELECT name FROM {table} LIMIT 1")).await); + let distinct_names = scalar_count( + &run( + &runtime, + &format!("SELECT count(DISTINCT name) FROM {table}"), + ) + .await, + ); + assert_eq!( + distinct_names, 1, + "UPDATE without WHERE should set every row" + ); + let any_name = + scalar_string(&run(&runtime, &format!("SELECT name FROM {table} LIMIT 1")).await); assert_eq!(any_name, "all", "every row should have the updated value"); // ALTER TABLE schema evolution (on the independent `copy` table, which still // holds the original rows (1,'a'),(2,'b')). // ADD COLUMN: existing rows read NULL. run(&runtime, &format!("ALTER TABLE {copy} ADD COLUMN age INT")).await; - let non_null_age = scalar_count(&run(&runtime, &format!("SELECT count(age) FROM {copy}")).await); - assert_eq!(non_null_age, 0, "existing rows must read NULL for a new column"); + let non_null_age = + scalar_count(&run(&runtime, &format!("SELECT count(age) FROM {copy}")).await); + assert_eq!( + non_null_age, 0, + "existing rows must read NULL for a new column" + ); // New rows can populate the new column. run(&runtime, &format!("INSERT INTO {copy} VALUES (3, 'c', 30)")).await; // ALTER COLUMN TYPE: int -> bigint (a safe promotion). - run(&runtime, &format!("ALTER TABLE {copy} ALTER COLUMN age TYPE BIGINT")).await; + run( + &runtime, + &format!("ALTER TABLE {copy} ALTER COLUMN age TYPE BIGINT"), + ) + .await; let age = scalar_count(&run(&runtime, &format!("SELECT age FROM {copy} WHERE id = 3")).await); assert_eq!(age, 30, "inserted value survives the type promotion"); // RENAME COLUMN: values preserved under the new name. - run(&runtime, &format!("ALTER TABLE {copy} RENAME COLUMN name TO label")).await; - let label = scalar_string(&run(&runtime, &format!("SELECT label FROM {copy} WHERE id = 1")).await); + run( + &runtime, + &format!("ALTER TABLE {copy} RENAME COLUMN name TO label"), + ) + .await; + let label = + scalar_string(&run(&runtime, &format!("SELECT label FROM {copy} WHERE id = 1")).await); assert_eq!(label, "a", "renamed column keeps its values"); // DROP COLUMN. run(&runtime, &format!("ALTER TABLE {copy} DROP COLUMN label")).await; @@ -120,13 +175,24 @@ async fn iceberg_create_insert_select_ctas_drop() { assert_eq!(names, vec!["id", "age"], "dropped column should be gone"); // A narrowing type change is rejected. let narrow = runtime - .run_query(beacon_core::query::Query::sql(format!("ALTER TABLE {copy} ALTER COLUMN age TYPE INT")), beacon_core::AuthIdentity::system()) + .run_query( + beacon_core::query::Query::sql(format!("ALTER TABLE {copy} ALTER COLUMN age TYPE INT")), + beacon_core::AuthIdentity::system(), + ) .await; assert!(narrow.is_err(), "narrowing type change should be rejected"); // Restore a known state for the DELETE cases below. - run(&runtime, &format!("UPDATE {table} SET name = 'a' WHERE id = 1")).await; - run(&runtime, &format!("UPDATE {table} SET name = 'b' WHERE id = 2")).await; + run( + &runtime, + &format!("UPDATE {table} SET name = 'a' WHERE id = 1"), + ) + .await; + run( + &runtime, + &format!("UPDATE {table} SET name = 'b' WHERE id = 2"), + ) + .await; // DELETE WHERE: remove one row, the other survives unchanged. run(&runtime, &format!("DELETE FROM {table} WHERE id = 1")).await; @@ -137,28 +203,45 @@ async fn iceberg_create_insert_select_ctas_drop() { // DELETE all rows. run(&runtime, &format!("DELETE FROM {table}")).await; - let after_delete_all = scalar_count(&run(&runtime, &format!("SELECT count(*) FROM {table}")).await); - assert_eq!(after_delete_all, 0, "DELETE without WHERE should empty the table"); + let after_delete_all = + scalar_count(&run(&runtime, &format!("SELECT count(*) FROM {table}")).await); + assert_eq!( + after_delete_all, 0, + "DELETE without WHERE should empty the table" + ); // DELETE on a non-Iceberg relation must be rejected. let view_name = format!("{table}_view"); - run(&runtime, &format!("CREATE VIEW {view_name} AS SELECT 1 AS id")).await; + run( + &runtime, + &format!("CREATE VIEW {view_name} AS SELECT 1 AS id"), + ) + .await; let delete_view = runtime - .run_query(beacon_core::query::Query::sql(format!("DELETE FROM {view_name} WHERE id = 1")), beacon_core::AuthIdentity::system()) + .run_query( + beacon_core::query::Query::sql(format!("DELETE FROM {view_name} WHERE id = 1")), + beacon_core::AuthIdentity::system(), + ) .await; assert!( delete_view.is_err(), "DELETE on a non-Iceberg table should error" ); let update_view = runtime - .run_query(beacon_core::query::Query::sql(format!("UPDATE {view_name} SET id = 2 WHERE id = 1")), beacon_core::AuthIdentity::system()) + .run_query( + beacon_core::query::Query::sql(format!("UPDATE {view_name} SET id = 2 WHERE id = 1")), + beacon_core::AuthIdentity::system(), + ) .await; assert!( update_view.is_err(), "UPDATE on a non-Iceberg table should error" ); let alter_view = runtime - .run_query(beacon_core::query::Query::sql(format!("ALTER TABLE {view_name} ADD COLUMN x INT")), beacon_core::AuthIdentity::system()) + .run_query( + beacon_core::query::Query::sql(format!("ALTER TABLE {view_name} ADD COLUMN x INT")), + beacon_core::AuthIdentity::system(), + ) .await; assert!( alter_view.is_err(), @@ -172,7 +255,10 @@ async fn iceberg_create_insert_select_ctas_drop() { // Dropped tables should no longer be queryable. let err = runtime - .run_query(beacon_core::query::Query::sql(format!("SELECT count(*) FROM {table}")), beacon_core::AuthIdentity::system()) + .run_query( + beacon_core::query::Query::sql(format!("SELECT count(*) FROM {table}")), + beacon_core::AuthIdentity::system(), + ) .await; assert!(err.is_err(), "querying a dropped table should error"); } diff --git a/beacon-core/tests/lance_tables.rs b/beacon-core/tests/lance_tables.rs index ceab3f02..53d11d46 100644 --- a/beacon-core/tests/lance_tables.rs +++ b/beacon-core/tests/lance_tables.rs @@ -10,7 +10,10 @@ use futures::TryStreamExt; /// Run SQL as a super-user (DDL/DML allowed) and collect the result batches. async fn run(runtime: &Runtime, sql: &str) -> Vec { runtime - .run_query(beacon_core::query::Query::sql(sql.to_string()), beacon_core::AuthIdentity::system()) + .run_query( + beacon_core::query::Query::sql(sql.to_string()), + beacon_core::AuthIdentity::system(), + ) .await .unwrap_or_else(|error| panic!("SQL failed to plan/execute: {sql}\n{error}")) .into_record_stream() @@ -54,15 +57,27 @@ async fn lance_create_insert_update_delete() { ) .await; - run(&runtime, &format!("CREATE TABLE {table} (id BIGINT, name VARCHAR)")).await; - run(&runtime, &format!("INSERT INTO {table} VALUES (1, 'a'), (2, 'b'), (3, 'c')")).await; + run( + &runtime, + &format!("CREATE TABLE {table} (id BIGINT, name VARCHAR)"), + ) + .await; + run( + &runtime, + &format!("INSERT INTO {table} VALUES (1, 'a'), (2, 'b'), (3, 'c')"), + ) + .await; assert_eq!( scalar_count(&run(&runtime, &format!("SELECT count(*) FROM {table}")).await), 3 ); // UPDATE WHERE: only the matching row changes; the others are untouched. - run(&runtime, &format!("UPDATE {table} SET name = 'Z' WHERE id = 2")).await; + run( + &runtime, + &format!("UPDATE {table} SET name = 'Z' WHERE id = 2"), + ) + .await; assert_eq!( scalar_string(&run(&runtime, &format!("SELECT name FROM {table} WHERE id = 2")).await), "Z" @@ -82,7 +97,11 @@ async fn lance_create_insert_update_delete() { run(&runtime, &format!("UPDATE {table} SET name = 'all'")).await; assert_eq!( scalar_count( - &run(&runtime, &format!("SELECT count(DISTINCT name) FROM {table}")).await + &run( + &runtime, + &format!("SELECT count(DISTINCT name) FROM {table}") + ) + .await ), 1, "UPDATE without WHERE should set every row" @@ -97,7 +116,11 @@ async fn lance_create_insert_update_delete() { ); assert_eq!( scalar_count( - &run(&runtime, &format!("SELECT id FROM {table} ORDER BY id LIMIT 1")).await + &run( + &runtime, + &format!("SELECT id FROM {table} ORDER BY id LIMIT 1") + ) + .await ), 2, "the smallest surviving id should be 2" diff --git a/beacon-core/tests/output_on_non_select.rs b/beacon-core/tests/output_on_non_select.rs index adfb68d2..e516e729 100644 --- a/beacon-core/tests/output_on_non_select.rs +++ b/beacon-core/tests/output_on_non_select.rs @@ -26,7 +26,10 @@ fn csv_query(sql: &str) -> Query { /// Run a CSV-output query expecting it to fail, returning the error message. /// (`QueryResult` is not `Debug`, so `Result::expect_err` is unavailable.) async fn expect_output_error(runtime: &Runtime, sql: &str) -> String { - match runtime.run_query(csv_query(sql), beacon_core::AuthIdentity::system()).await { + match runtime + .run_query(csv_query(sql), beacon_core::AuthIdentity::system()) + .await + { Ok(_) => panic!("expected an error for: {sql}"), Err(error) => error.to_string(), } @@ -39,7 +42,10 @@ async fn output_on_non_row_producing_statement_is_a_clear_error() { let runtime = boot().await; let table = format!("output_guard_{}", std::process::id()); let _ = runtime - .run_query(Query::sql(format!("DROP TABLE IF EXISTS {table}")), beacon_core::AuthIdentity::system()) + .run_query( + Query::sql(format!("DROP TABLE IF EXISTS {table}")), + beacon_core::AuthIdentity::system(), + ) .await; // CREATE TABLE (DDL) and SET (statement) both return no rows. @@ -70,7 +76,10 @@ async fn output_on_non_row_producing_statement_is_a_clear_error() { ); let _ = runtime - .run_query(Query::sql(format!("DROP TABLE IF EXISTS {table}")), beacon_core::AuthIdentity::system()) + .run_query( + Query::sql(format!("DROP TABLE IF EXISTS {table}")), + beacon_core::AuthIdentity::system(), + ) .await; } @@ -79,7 +88,10 @@ async fn output_on_non_row_producing_statement_is_a_clear_error() { async fn output_on_select_still_succeeds() { let runtime = boot().await; let result = runtime - .run_query(csv_query("SELECT 1 AS a"), beacon_core::AuthIdentity::system()) + .run_query( + csv_query("SELECT 1 AS a"), + beacon_core::AuthIdentity::system(), + ) .await .expect("SELECT with output should succeed"); assert!( diff --git a/beacon-data-lake/src/crawler/definition.rs b/beacon-data-lake/src/crawler/definition.rs index 77cda3ff..159a2883 100644 --- a/beacon-data-lake/src/crawler/definition.rs +++ b/beacon-data-lake/src/crawler/definition.rs @@ -99,8 +99,9 @@ impl CrawlerDefinition { }); let detect_partitions = match with.get("detect_partitions") { - Some(v) => parse_bool(v) - .ok_or_else(|| format!("invalid detect_partitions value '{v}'"))?, + Some(v) => { + parse_bool(v).ok_or_else(|| format!("invalid detect_partitions value '{v}'"))? + } None => true, }; @@ -216,7 +217,10 @@ mod tests { assert_eq!(def.schedule(), Some(Duration::from_secs(900))); assert_eq!(def.table_naming, TableNaming::CrawlerPrefixed); // Non-control keys are forwarded verbatim. - assert_eq!(def.options.get("read_dimensions").map(String::as_str), Some("lat,lon")); + assert_eq!( + def.options.get("read_dimensions").map(String::as_str), + Some("lat,lon") + ); assert!(!def.options.contains_key("format")); assert!(!def.options.contains_key("schedule")); } diff --git a/beacon-data-lake/src/crawler/discovery.rs b/beacon-data-lake/src/crawler/discovery.rs index 85edac85..8ef78fac 100644 --- a/beacon-data-lake/src/crawler/discovery.rs +++ b/beacon-data-lake/src/crawler/discovery.rs @@ -195,7 +195,13 @@ pub fn group_into_tables( fn slugify(input: &str) -> String { let s: String = input .chars() - .map(|c| if c.is_ascii_alphanumeric() { c.to_ascii_lowercase() } else { '_' }) + .map(|c| { + if c.is_ascii_alphanumeric() { + c.to_ascii_lowercase() + } else { + '_' + } + }) .collect(); s.trim_matches('_').to_string() } @@ -276,7 +282,10 @@ mod tests { assert_eq!(cands.len(), 1); let c = &cands[0]; assert_eq!(c.base, "argo"); - assert_eq!(c.partition_cols, vec!["year".to_string(), "month".to_string()]); + assert_eq!( + c.partition_cols, + vec!["year".to_string(), "month".to_string()] + ); assert!(c.recursive); assert_eq!(c.file_count, 2); assert_eq!(c.location(), "argo/**/*.parquet"); @@ -284,7 +293,10 @@ mod tests { #[test] fn flat_layout_has_no_partitions() { - let datasets = vec![ds("argo/a.parquet", "parquet"), ds("argo/b.parquet", "parquet")]; + let datasets = vec![ + ds("argo/a.parquet", "parquet"), + ds("argo/b.parquet", "parquet"), + ]; let (cands, _) = group_into_tables(&datasets, &def()); assert_eq!(cands.len(), 1); assert_eq!(cands[0].base, "argo"); @@ -362,7 +374,10 @@ mod tests { recursive: false, file_count: 1, }]; - assert_eq!(assign_table_names(&cands, &def()), vec!["argo_floats".to_string()]); + assert_eq!( + assign_table_names(&cands, &def()), + vec!["argo_floats".to_string()] + ); let mut prefixed = def(); prefixed.name = "ocean".to_string(); @@ -394,6 +409,9 @@ mod tests { let names = assign_table_names(&cands, &def()); assert_eq!(names[0], "data"); assert_ne!(names[1], "data"); - assert_eq!(names.iter().collect::>().len(), 2); + assert_eq!( + names.iter().collect::>().len(), + 2 + ); } } diff --git a/beacon-data-lake/src/crawler/engine.rs b/beacon-data-lake/src/crawler/engine.rs index 50a19114..6154ddd1 100644 --- a/beacon-data-lake/src/crawler/engine.rs +++ b/beacon-data-lake/src/crawler/engine.rs @@ -19,9 +19,9 @@ use beacon_datafusion_ext::table_ext::{ExternalTable, ExternalTableDefinition, T use datafusion::prelude::SessionContext; use serde::{Deserialize, Serialize}; -use crate::{list_datasets, DATASETS_OBJECT_STORE_URL}; +use crate::{DATASETS_OBJECT_STORE_URL, list_datasets}; -use super::definition::{CrawlerDefinition, CRAWLER_OWNER_OPTION}; +use super::definition::{CRAWLER_OWNER_OPTION, CrawlerDefinition}; use super::discovery::{assign_table_names, group_into_tables}; /// Outcome of a single crawl, suitable for logging or returning over the API. @@ -75,7 +75,10 @@ impl CrawlEngine { .runtime_env() .object_store(&*DATASETS_OBJECT_STORE_URL) .map_err(|e| { - anyhow::anyhow!("crawler '{}' could not resolve datasets store: {e}", def.name) + anyhow::anyhow!( + "crawler '{}' could not resolve datasets store: {e}", + def.name + ) })?; let datasets = list_datasets( &self.session_ctx, diff --git a/beacon-data-lake/src/crawler/manager.rs b/beacon-data-lake/src/crawler/manager.rs index 27681785..c7f6ff8e 100644 --- a/beacon-data-lake/src/crawler/manager.rs +++ b/beacon-data-lake/src/crawler/manager.rs @@ -9,11 +9,11 @@ use std::sync::{Arc, OnceLock, Weak}; use std::time::Duration; use beacon_datafusion_ext::format_ext::FileFormatFactoryExt; -use beacon_object_storage::event::ObjectEvent; use beacon_object_storage::DatasetsStore; +use beacon_object_storage::event::ObjectEvent; use datafusion::prelude::SessionContext; use parking_lot::Mutex; -use tokio::sync::{broadcast, Mutex as AsyncMutex}; +use tokio::sync::{Mutex as AsyncMutex, broadcast}; use tokio::task::JoinHandle; use crate::TABLES_OBJECT_STORE_URL; diff --git a/beacon-data-lake/src/crawler/mod.rs b/beacon-data-lake/src/crawler/mod.rs index af648065..dde3ae91 100644 --- a/beacon-data-lake/src/crawler/mod.rs +++ b/beacon-data-lake/src/crawler/mod.rs @@ -15,8 +15,8 @@ pub mod engine; pub mod manager; pub mod persistence; -pub use definition::{CrawlerDefinition, TableNaming, CRAWLER_OWNER_OPTION}; -pub use discovery::{assign_table_names, group_into_tables, CandidateTable}; +pub use definition::{CRAWLER_OWNER_OPTION, CrawlerDefinition, TableNaming}; +pub use discovery::{CandidateTable, assign_table_names, group_into_tables}; pub use engine::{CrawlEngine, CrawlReport}; -pub use manager::{new_crawler_manager_handle, CrawlerManager, CrawlerManagerHandle}; +pub use manager::{CrawlerManager, CrawlerManagerHandle, new_crawler_manager_handle}; pub use persistence::CrawlerPersistence; diff --git a/beacon-data-lake/src/crawler/persistence.rs b/beacon-data-lake/src/crawler/persistence.rs index d4f51253..4c3c9e9f 100644 --- a/beacon-data-lake/src/crawler/persistence.rs +++ b/beacon-data-lake/src/crawler/persistence.rs @@ -11,7 +11,7 @@ use datafusion::{ error::DataFusionError, execution::object_store::ObjectStoreUrl, prelude::SessionContext, }; use futures::StreamExt; -use object_store::{path::Path, ObjectStore, ObjectStoreExt}; +use object_store::{ObjectStore, ObjectStoreExt, path::Path}; use super::definition::CrawlerDefinition; diff --git a/beacon-data-lake/src/files/mod.rs b/beacon-data-lake/src/files/mod.rs index fb9b885b..14ef3915 100644 --- a/beacon-data-lake/src/files/mod.rs +++ b/beacon-data-lake/src/files/mod.rs @@ -55,7 +55,9 @@ pub async fn list_datasets( let listing_url = create_listing_url(pattern.unwrap_or_else(|| "*".to_string()))?; let mut objects = Vec::new(); - let mut entry_stream = listing_url.list_all_files(&state, &object_store, "").await?; + let mut entry_stream = listing_url + .list_all_files(&state, &object_store, "") + .await?; while let Some(entry) = entry_stream.next().await { if let Ok(entry) = entry { @@ -147,7 +149,12 @@ pub async fn list_dataset_schema( // in the format list. let file_format_factory = file_formats .iter() - .find(|factory| factory.file_extensions().iter().any(|ext| ext == &extension)) + .find(|factory| { + factory + .file_extensions() + .iter() + .any(|ext| ext == &extension) + }) .map(|factory| factory.clone() as Arc) .or_else(|| session_state.get_file_format_factory(&extension)) .ok_or_else(|| { diff --git a/beacon-data-lake/src/lib.rs b/beacon-data-lake/src/lib.rs index 27b2e71b..d4f38aa2 100644 --- a/beacon-data-lake/src/lib.rs +++ b/beacon-data-lake/src/lib.rs @@ -15,12 +15,12 @@ pub use files::temp_output_file::TempOutputFile; pub use files::{create_listing_url, create_temp_output_file, list_dataset_schema, list_datasets}; pub use table_runtime::init_tables; pub use table_runtime::persistent_schema_provider::PersistentSchemaProvider; -pub use table_runtime::schema_persistence::{definition_from_provider, SchemaPersistenceService}; +pub use table_runtime::schema_persistence::{SchemaPersistenceService, definition_from_provider}; pub mod prelude { pub use super::files::*; pub use super::{ - definition_from_provider, init_tables, register_object_stores, PersistentSchemaProvider, + PersistentSchemaProvider, definition_from_provider, init_tables, register_object_stores, }; } diff --git a/beacon-data-lake/src/table_runtime/persistent_schema_provider.rs b/beacon-data-lake/src/table_runtime/persistent_schema_provider.rs index a05d5830..4698d0f2 100644 --- a/beacon-data-lake/src/table_runtime/persistent_schema_provider.rs +++ b/beacon-data-lake/src/table_runtime/persistent_schema_provider.rs @@ -146,7 +146,7 @@ mod tests { use super::*; use datafusion::datasource::ViewTable; use futures::StreamExt; - use object_store::{memory::InMemory, path::Path, ObjectStore, ObjectStoreExt}; + use object_store::{ObjectStore, ObjectStoreExt, memory::InMemory, path::Path}; use url::Url; // The persistence side effect runs the async store I/O via `block_in_place`, @@ -184,7 +184,10 @@ mod tests { .register_table("v".to_string(), view(&ctx, "SELECT 1 AS x").await) .expect("registration should succeed"); - assert!(previous.is_none(), "first registration has no previous table"); + assert!( + previous.is_none(), + "first registration has no previous table" + ); assert!(provider.table_exist("v")); assert!( store.get(&Path::from("v/table.json")).await.is_ok(), @@ -206,7 +209,10 @@ mod tests { .register_table("v".to_string(), view(&ctx, "SELECT 2 AS y").await) .expect("re-registering an existing name should overwrite, not error"); - assert!(previous.is_some(), "overwrite returns the replaced provider"); + assert!( + previous.is_some(), + "overwrite returns the replaced provider" + ); let table = provider .table("v") .await diff --git a/beacon-data-lake/src/table_runtime/schema_persistence.rs b/beacon-data-lake/src/table_runtime/schema_persistence.rs index e538046b..a264041c 100644 --- a/beacon-data-lake/src/table_runtime/schema_persistence.rs +++ b/beacon-data-lake/src/table_runtime/schema_persistence.rs @@ -1,5 +1,5 @@ -use std::sync::Arc; use object_store::ObjectStoreExt; +use std::sync::Arc; use beacon_datafusion_ext::table_ext::{ ExternalTable, MaterializedView, TableDefinition, ViewTableDefinition, @@ -220,7 +220,10 @@ pub fn definition_from_provider( table_name: &str, table: &dyn TableProvider, ) -> datafusion::error::Result> { - if let Some(table) = table.as_any().downcast_ref::() { + if let Some(table) = table + .as_any() + .downcast_ref::() + { Ok(Arc::new(table.definition().clone())) } else if let Some(table) = table.as_any().downcast_ref::() { Ok(Arc::new(table.definition().clone())) @@ -230,7 +233,10 @@ pub fn definition_from_provider( Ok(Arc::new(table.definition().clone())) } else if let Some(definition) = beacon_datafusion_ext::remote::remote_table_definition(table) { Ok(Arc::new(definition)) - } else if let Some(table) = table.as_any().downcast_ref::() { + } else if let Some(table) = table + .as_any() + .downcast_ref::() + { Ok(Arc::new(table.definition().clone())) } else if let Some(definition) = beacon_sql_databases::sql_database_table_definition(table) { Ok(Arc::new(definition)) @@ -351,11 +357,13 @@ mod tests { let (service, _ctx, table_store, _url) = test_service(); // Missing sidecar reads as None. - assert!(service - .load_table_extensions_json("obs") - .await - .expect("load should succeed") - .is_none()); + assert!( + service + .load_table_extensions_json("obs") + .await + .expect("load should succeed") + .is_none() + ); // Persist then load returns the stored JSON. let payload = r#"{"mcp":{"enabled":true}}"#.to_string(); @@ -371,21 +379,25 @@ mod tests { .as_deref(), Some(payload.as_str()) ); - assert!(table_store - .get(&Path::from("obs/extensions.json")) - .await - .is_ok()); + assert!( + table_store + .get(&Path::from("obs/extensions.json")) + .await + .is_ok() + ); // Explicit removal clears it (and is a no-op when already absent). service .remove_table_extensions_json("obs") .await .expect("remove should succeed"); - assert!(service - .load_table_extensions_json("obs") - .await - .expect("load should succeed") - .is_none()); + assert!( + service + .load_table_extensions_json("obs") + .await + .expect("load should succeed") + .is_none() + ); service .remove_table_extensions_json("obs") .await @@ -400,10 +412,12 @@ mod tests { .remove_persisted_table("obs") .await .expect("table removal should succeed"); - assert!(service - .load_table_extensions_json("obs") - .await - .expect("load should succeed") - .is_none()); + assert!( + service + .load_table_extensions_json("obs") + .await + .expect("load should succeed") + .is_none() + ); } } diff --git a/beacon-datafusion-ext/src/listing_table_factory_ext.rs b/beacon-datafusion-ext/src/listing_table_factory_ext.rs index 217b1100..4d2c84f3 100644 --- a/beacon-datafusion-ext/src/listing_table_factory_ext.rs +++ b/beacon-datafusion-ext/src/listing_table_factory_ext.rs @@ -16,7 +16,9 @@ use datafusion::{ catalog::{Session, TableProvider, TableProviderFactory}, }; -use crate::table_ext::{ExternalTable, ExternalTableDefinition, ExternalTableRebuild, build_listing_table}; +use crate::table_ext::{ + ExternalTable, ExternalTableDefinition, ExternalTableRebuild, build_listing_table, +}; type PartitionCols = Vec<(String, DataType)>; @@ -123,14 +125,11 @@ impl TableProviderFactory for ListingTableFactoryExt { // Resolve the runtime's datasets store from the session config extension // so self-refreshing tables can subscribe to its change events. - let datasets_store = self - .session_ctx - .upgrade() - .and_then(|ctx| { - ctx.state() - .config() - .get_extension::() - }); + let datasets_store = self.session_ctx.upgrade().and_then(|ctx| { + ctx.state() + .config() + .get_extension::() + }); let events = crate::table_ext::datasets_store_events(&self.store_url, datasets_store); let external_table = ExternalTable::new_self_refreshing( diff --git a/beacon-datafusion-ext/src/remote/executor.rs b/beacon-datafusion-ext/src/remote/executor.rs index 4a54f46a..7336b455 100644 --- a/beacon-datafusion-ext/src/remote/executor.rs +++ b/beacon-datafusion-ext/src/remote/executor.rs @@ -45,10 +45,7 @@ impl BeaconFlightSqlExecutor { /// Fetch a remote table's schema without transferring data, via a /// `LIMIT 0` query whose `FlightInfo` carries the IPC-encoded schema. - pub async fn fetch_schema( - connection: &RemoteConnection, - table: &str, - ) -> DFResult { + pub async fn fetch_schema(connection: &RemoteConnection, table: &str) -> DFResult { let mut client = connection.connect().await.map_err(remote_err)?; let info = client .execute(format!("SELECT * FROM {table} LIMIT 0"), None) diff --git a/beacon-datafusion-ext/src/remote/mod.rs b/beacon-datafusion-ext/src/remote/mod.rs index 210f2a8a..8a39038c 100644 --- a/beacon-datafusion-ext/src/remote/mod.rs +++ b/beacon-datafusion-ext/src/remote/mod.rs @@ -28,6 +28,9 @@ pub fn remote_table_definition(provider: &dyn TableProvider) -> Option()?; let source = adaptor.source.as_any().downcast_ref::()?; - let table = source.table.as_any().downcast_ref::()?; + let table = source + .table + .as_any() + .downcast_ref::()?; Some(table.definition().clone()) } diff --git a/beacon-datafusion-ext/src/stats_cache.rs b/beacon-datafusion-ext/src/stats_cache.rs index 08b0bd22..c1a23dab 100644 --- a/beacon-datafusion-ext/src/stats_cache.rs +++ b/beacon-datafusion-ext/src/stats_cache.rs @@ -75,7 +75,11 @@ impl BeaconFileStatisticsCache { self.inner .iter() .map(|(path, cached)| { - ((*path).clone(), cached.meta.clone(), Arc::clone(&cached.statistics)) + ( + (*path).clone(), + cached.meta.clone(), + Arc::clone(&cached.statistics), + ) }) .collect() } @@ -97,7 +101,10 @@ impl BeaconFileStatisticsCache { value: Arc, e: &ObjectMeta, ) -> Option> { - let old = self.inner.get(key).map(|cached| Arc::clone(&cached.statistics)); + let old = self + .inner + .get(key) + .map(|cached| Arc::clone(&cached.statistics)); self.inner .insert(key.clone(), CachedFileMetadata::new(e.clone(), value, None)); old @@ -187,25 +194,49 @@ mod tests { let p = Path::from("a/b.parquet"); // Nothing cached yet. - assert!(cache.get_with_extra(&p, &meta("a/b.parquet", 100)).is_none()); + assert!( + cache + .get_with_extra(&p, &meta("a/b.parquet", 100)) + .is_none() + ); // First insert returns no previous value. - assert!(cache.put_with_extra(&p, stats(), &meta("a/b.parquet", 100)).is_none()); + assert!( + cache + .put_with_extra(&p, stats(), &meta("a/b.parquet", 100)) + .is_none() + ); // Matching size + last_modified => hit. - assert!(cache.get_with_extra(&p, &meta("a/b.parquet", 100)).is_some()); + assert!( + cache + .get_with_extra(&p, &meta("a/b.parquet", 100)) + .is_some() + ); // Size changed => the cached entry is treated as stale. - assert!(cache.get_with_extra(&p, &meta("a/b.parquet", 200)).is_none()); + assert!( + cache + .get_with_extra(&p, &meta("a/b.parquet", 200)) + .is_none() + ); } #[test] fn put_with_extra_returns_the_previous_statistics() { let cache = BeaconFileStatisticsCache::with_capacity(8); let p = Path::from("c.parquet"); - assert!(cache.put_with_extra(&p, stats(), &meta("c.parquet", 1)).is_none()); + assert!( + cache + .put_with_extra(&p, stats(), &meta("c.parquet", 1)) + .is_none() + ); // A second put for the same key reports the prior value. - assert!(cache.put_with_extra(&p, stats(), &meta("c.parquet", 2)).is_some()); + assert!( + cache + .put_with_extra(&p, stats(), &meta("c.parquet", 2)) + .is_some() + ); } #[test] diff --git a/beacon-datafusion-ext/src/table_ext.rs b/beacon-datafusion-ext/src/table_ext.rs index 96a542a7..0c73a62a 100644 --- a/beacon-datafusion-ext/src/table_ext.rs +++ b/beacon-datafusion-ext/src/table_ext.rs @@ -793,7 +793,10 @@ impl TableDefinition for ViewTableDefinition { LogicalPlan::Ddl(DdlStatement::CreateView(plan)) => plan.input.as_ref().clone(), plan => plan, }; - Ok(Arc::new(ViewTable::new(input, Some(self.definition.clone())))) + Ok(Arc::new(ViewTable::new( + input, + Some(self.definition.clone()), + ))) } fn table_name(&self) -> &str { @@ -926,12 +929,13 @@ impl TableDefinition for MaterializedViewDefinition { _data_store_url: &ObjectStoreUrl, ) -> anyhow::Result> { let session_state = context.state(); - let file_format_factory = session_state - .get_file_format_factory("parquet") - .ok_or(config_datafusion_err!( - "Unable to build materialized view '{}': parquet FileFormat not found.", - self.name - ))?; + let file_format_factory = + session_state + .get_file_format_factory("parquet") + .ok_or(config_datafusion_err!( + "Unable to build materialized view '{}': parquet FileFormat not found.", + self.name + ))?; let file_format = file_format_factory.create(&session_state, &std::collections::HashMap::new())?; @@ -993,7 +997,11 @@ mod self_refresh_tests { fn write_parquet_i64(disk_path: &std::path::Path, values: &[i64]) { std::fs::create_dir_all(disk_path.parent().expect("path has parent")) .expect("create parent dirs"); - let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Int64, false)])); + let schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int64, + false, + )])); let batch = RecordBatch::try_new( schema.clone(), vec![Arc::new(Int64Array::from(values.to_vec()))], @@ -1055,7 +1063,13 @@ mod self_refresh_tests { options: HashMap::new(), if_not_exists: false, }; - ExternalTable::new_self_refreshing(definition, initial, rebuild, Arc::downgrade(ctx), events) + ExternalTable::new_self_refreshing( + definition, + initial, + rebuild, + Arc::downgrade(ctx), + events, + ) } #[tokio::test(flavor = "multi_thread")] @@ -1065,7 +1079,8 @@ mod self_refresh_tests { let ctx = ctx_with_datasets(dir.path()); let external = build_external(&ctx, None).await; - ctx.register_table("obs", Arc::new(external.clone())).unwrap(); + ctx.register_table("obs", Arc::new(external.clone())) + .unwrap(); assert_eq!(count_rows(&ctx).await, 1); // A new file appears; manual refresh re-lists and picks it up. diff --git a/beacon-file-formats/beacon-arrow-atlas/src/backend.rs b/beacon-file-formats/beacon-arrow-atlas/src/backend.rs index 4e53d5b6..93d1c81c 100644 --- a/beacon-file-formats/beacon-arrow-atlas/src/backend.rs +++ b/beacon-file-formats/beacon-arrow-atlas/src/backend.rs @@ -46,11 +46,7 @@ macro_rules! impl_atlas_readable_passthrough { .read_array::<$ty>(array_name, start, shape) .await .map_err(|e| { - anyhow::anyhow!( - "Failed to read atlas array '{}': {}", - array_name, - e - ) + anyhow::anyhow!("Failed to read atlas array '{}': {}", array_name, e) })? .ok_or_else(|| { anyhow::anyhow!( @@ -186,13 +182,17 @@ impl ArrayBackend for AtlasArrayBackend { } async fn read_subset(&self, subset: ArraySubset) -> anyhow::Result> { - let view = self.atlas.open_dataset(&self.dataset_name).await.map_err(|e| { - anyhow::anyhow!( - "Failed to open atlas dataset '{}': {}", - self.dataset_name, - e - ) - })?; + let view = self + .atlas + .open_dataset(&self.dataset_name) + .await + .map_err(|e| { + anyhow::anyhow!( + "Failed to open atlas dataset '{}': {}", + self.dataset_name, + e + ) + })?; T::read(&view, &self.array_name, subset.start, subset.shape).await } } @@ -304,7 +304,10 @@ mod tests { vec![4], Some(-1.0f32), ); - assert_eq!( as ArrayBackend>::shape(&backend), vec![4]); + assert_eq!( + as ArrayBackend>::shape(&backend), + vec![4] + ); assert_eq!( as ArrayBackend>::dimensions(&backend), vec!["obs".to_string()] diff --git a/beacon-file-formats/beacon-arrow-atlas/src/compat.rs b/beacon-file-formats/beacon-arrow-atlas/src/compat.rs index aa81be01..4518a490 100644 --- a/beacon-file-formats/beacon-arrow-atlas/src/compat.rs +++ b/beacon-file-formats/beacon-arrow-atlas/src/compat.rs @@ -181,10 +181,18 @@ pub fn array_to_nd_array( /// Convert an atlas attribute value into a rank-0 ND array. pub fn attribute_to_nd_array(_name: &str, attr: Attr) -> anyhow::Result> { match attr { - Attr::Bool(v) => Ok(Arc::new(NdArray::new_with_backend(AttributeBackend::new(v))?)), - Attr::Int64(v) => Ok(Arc::new(NdArray::new_with_backend(AttributeBackend::new(v))?)), - Attr::Float64(v) => Ok(Arc::new(NdArray::new_with_backend(AttributeBackend::new(v))?)), - Attr::String(v) => Ok(Arc::new(NdArray::new_with_backend(AttributeBackend::new(v))?)), + Attr::Bool(v) => Ok(Arc::new(NdArray::new_with_backend(AttributeBackend::new( + v, + ))?)), + Attr::Int64(v) => Ok(Arc::new(NdArray::new_with_backend(AttributeBackend::new( + v, + ))?)), + Attr::Float64(v) => Ok(Arc::new(NdArray::new_with_backend(AttributeBackend::new( + v, + ))?)), + Attr::String(v) => Ok(Arc::new(NdArray::new_with_backend(AttributeBackend::new( + v, + ))?)), Attr::TimestampNanoseconds(v) => Ok(Arc::new(NdArray::new_with_backend( AttributeBackend::new(TimestampNanosecond(v)), )?)), @@ -288,7 +296,10 @@ mod tests { let nd = attribute_to_nd_array("flag", Attr::Bool(true)).expect("convert"); assert_eq!(nd.datatype(), NdArrayDataType::Bool); assert!(nd.shape().is_empty()); - let typed = nd.as_any().downcast_ref::>().expect("downcast"); + let typed = nd + .as_any() + .downcast_ref::>() + .expect("downcast"); assert_eq!(typed.clone_into_raw_vec().await, vec![true]); } @@ -296,7 +307,10 @@ mod tests { async fn attribute_int64_round_trips() { let nd = attribute_to_nd_array("count", Attr::Int64(42)).expect("convert"); assert_eq!(nd.datatype(), NdArrayDataType::I64); - let typed = nd.as_any().downcast_ref::>().expect("downcast"); + let typed = nd + .as_any() + .downcast_ref::>() + .expect("downcast"); assert_eq!(typed.clone_into_raw_vec().await, vec![42i64]); } @@ -304,7 +318,10 @@ mod tests { async fn attribute_float64_round_trips() { let nd = attribute_to_nd_array("scale", Attr::Float64(1.5)).expect("convert"); assert_eq!(nd.datatype(), NdArrayDataType::F64); - let typed = nd.as_any().downcast_ref::>().expect("downcast"); + let typed = nd + .as_any() + .downcast_ref::>() + .expect("downcast"); assert_eq!(typed.clone_into_raw_vec().await, vec![1.5f64]); } diff --git a/beacon-file-formats/beacon-arrow-atlas/src/datafusion/cache.rs b/beacon-file-formats/beacon-arrow-atlas/src/datafusion/cache.rs index 89199576..aadd61d1 100644 --- a/beacon-file-formats/beacon-arrow-atlas/src/datafusion/cache.rs +++ b/beacon-file-formats/beacon-arrow-atlas/src/datafusion/cache.rs @@ -77,7 +77,10 @@ pub async fn get_or_open_atlas( cache .cache - .try_get_with(key, async move { reader::open_atlas_store(store, &path).await }) + .try_get_with( + key, + async move { reader::open_atlas_store(store, &path).await }, + ) .await .map_err(|e: Arc| { datafusion::error::DataFusionError::Execution(format!( @@ -92,7 +95,11 @@ mod tests { use crate::datafusion::test_support::{ensure_fixture, fixture_marker_object_meta, test_store}; use object_store::path::Path as OsPath; - fn marker_with(path: OsPath, last_modified: chrono::DateTime, size: u64) -> ObjectMeta { + fn marker_with( + path: OsPath, + last_modified: chrono::DateTime, + size: u64, + ) -> ObjectMeta { ObjectMeta { location: path, last_modified, diff --git a/beacon-file-formats/beacon-arrow-atlas/src/datafusion/mod.rs b/beacon-file-formats/beacon-arrow-atlas/src/datafusion/mod.rs index 5e80f511..7073dd98 100644 --- a/beacon-file-formats/beacon-arrow-atlas/src/datafusion/mod.rs +++ b/beacon-file-formats/beacon-arrow-atlas/src/datafusion/mod.rs @@ -641,7 +641,8 @@ mod tests { #[tokio::test] async fn factory_get_ext_returns_atlas() { let store = test_store().await; - let factory = AtlasFormatFactory::new(store, AtlasOptions::default(), AtlasConfig::default()); + let factory = + AtlasFormatFactory::new(store, AtlasOptions::default(), AtlasConfig::default()); // `get_ext` is the format identity used to register and resolve // external tables (`STORED AS ATLAS`), not the marker filename // (`atlas.json`). @@ -652,7 +653,8 @@ mod tests { #[tokio::test] async fn discover_datasets_emits_one_entry_per_atlas_dataset() { let store = test_store().await; - let factory = AtlasFormatFactory::new(store, AtlasOptions::default(), AtlasConfig::default()); + let factory = + AtlasFormatFactory::new(store, AtlasOptions::default(), AtlasConfig::default()); let objects = vec![fixture_marker_object_meta()]; let datasets = factory.discover_datasets(&objects).expect("discover"); @@ -671,7 +673,8 @@ mod tests { #[tokio::test] async fn discover_datasets_ignores_non_marker_objects() { let store = test_store().await; - let factory = AtlasFormatFactory::new(store, AtlasOptions::default(), AtlasConfig::default()); + let factory = + AtlasFormatFactory::new(store, AtlasOptions::default(), AtlasConfig::default()); let objects = vec![ObjectMeta { location: object_store::path::Path::from("some/other.nc"), last_modified: chrono::Utc::now(), @@ -762,8 +765,7 @@ mod tests { insert_op: datafusion::logical_expr::dml::InsertOp::Append, keep_partition_by_columns: false, file_extension: "atlas.json".to_string(), - file_output_mode: - datafusion::datasource::physical_plan::FileOutputMode::SingleFile, + file_output_mode: datafusion::datasource::physical_plan::FileOutputMode::SingleFile, }; let err = format .create_writer_physical_plan(input, &ctx.state(), conf, None) @@ -1000,7 +1002,8 @@ mod tests { async fn discover_datasets_finds_msgpack_zst_store() { ensure_msgpack_zst_fixture().await; let store = test_store().await; - let factory = AtlasFormatFactory::new(store, AtlasOptions::default(), AtlasConfig::default()); + let factory = + AtlasFormatFactory::new(store, AtlasOptions::default(), AtlasConfig::default()); let marker = ObjectMeta { location: OsPath::from(format!("{MSGPACK_ZST_FIXTURE_DIR}/atlas.msgpack.zst")), @@ -1115,18 +1118,21 @@ mod tests { .unwrap(); let mut temps: Vec = vec![]; for b in &batches { - let col = b - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); + let col = b.column(0).as_any().downcast_ref::().unwrap(); for i in 0..col.len() { - assert!(col.value(i) > 10.0, "every returned temperature must satisfy the predicate"); + assert!( + col.value(i) > 10.0, + "every returned temperature must satisfy the predicate" + ); temps.push(col.value(i)); } } temps.sort_by(|a, b| a.partial_cmp(b).unwrap()); - assert_eq!(temps, vec![20.0f32, 21.0, 22.0], "only summer's temperatures remain"); + assert_eq!( + temps, + vec![20.0f32, 21.0, 22.0], + "only summer's temperatures remain" + ); } } diff --git a/beacon-file-formats/beacon-arrow-atlas/src/datafusion/source.rs b/beacon-file-formats/beacon-arrow-atlas/src/datafusion/source.rs index 56073767..18bbc8b1 100644 --- a/beacon-file-formats/beacon-arrow-atlas/src/datafusion/source.rs +++ b/beacon-file-formats/beacon-arrow-atlas/src/datafusion/source.rs @@ -1,13 +1,13 @@ use std::collections::HashMap; use std::sync::Arc; +use crate::datafusion::cache::AtlasReaderCache; use arrow::{datatypes::SchemaRef, record_batch::RecordBatch}; use beacon_nd_array::arrow::{ batch::any_dataset_as_record_batch_stream, metrics::DatasetReadMetrics, pushdown_filter::PushdownFilter, }; use beacon_object_storage::DatasetsStore; -use crate::datafusion::cache::AtlasReaderCache; use datafusion::physical_expr_adapter::BatchAdapterFactory; use datafusion::{ common::Statistics, diff --git a/beacon-file-formats/beacon-arrow-atlas/src/reader.rs b/beacon-file-formats/beacon-arrow-atlas/src/reader.rs index 6fe7b60c..e2220308 100644 --- a/beacon-file-formats/beacon-arrow-atlas/src/reader.rs +++ b/beacon-file-formats/beacon-arrow-atlas/src/reader.rs @@ -54,7 +54,8 @@ pub async fn dataset_from_atlas( .await .map_err(|e| anyhow::anyhow!("Failed to open atlas dataset '{}': {}", dataset_name, e))?; - let included = |name: &str| projected_names.map_or(true, |names| names.iter().any(|n| n == name)); + let included = + |name: &str| projected_names.map_or(true, |names| names.iter().any(|n| n == name)); let mut arrays: IndexMap> = IndexMap::new(); diff --git a/beacon-file-formats/beacon-arrow-bbf/src/datafusion/mod.rs b/beacon-file-formats/beacon-arrow-bbf/src/datafusion/mod.rs index d1db847e..64479893 100644 --- a/beacon-file-formats/beacon-arrow-bbf/src/datafusion/mod.rs +++ b/beacon-file-formats/beacon-arrow-bbf/src/datafusion/mod.rs @@ -78,7 +78,9 @@ impl FileFormatFactory for BBFFormatFactory { if let Some(value) = format_options.get("split_streams_slice") { split_streams_slice = parse_bool_option("split_streams_slice", value)?; } - Ok(Arc::new(BBFFormat { split_streams_slice })) + Ok(Arc::new(BBFFormat { + split_streams_slice, + })) } fn as_any(&self) -> &dyn Any { diff --git a/beacon-file-formats/beacon-arrow-bbf/src/datafusion/opener.rs b/beacon-file-formats/beacon-arrow-bbf/src/datafusion/opener.rs index 6ffce79a..fdbe07b5 100644 --- a/beacon-file-formats/beacon-arrow-bbf/src/datafusion/opener.rs +++ b/beacon-file-formats/beacon-arrow-bbf/src/datafusion/opener.rs @@ -314,7 +314,8 @@ impl PruningStatistics for BBFPruningStatistics { // Return Null Array as all the values are null. If the column is not // in the table schema, there are no usable stats: return None so the // container is conservatively kept. - let null_array = new_null_array(self.table_schema.get(&column.name)?, self.num_containers); + let null_array = + new_null_array(self.table_schema.get(&column.name)?, self.num_containers); return Some(null_array); } @@ -342,7 +343,8 @@ impl PruningStatistics for BBFPruningStatistics { // Return Null Array as all the values are null. If the column is not // in the table schema, there are no usable stats: return None so the // container is conservatively kept. - let null_array = new_null_array(self.table_schema.get(&column.name)?, self.num_containers); + let null_array = + new_null_array(self.table_schema.get(&column.name)?, self.num_containers); return Some(null_array); } diff --git a/beacon-file-formats/beacon-arrow-bbf/src/datafusion/table_function.rs b/beacon-file-formats/beacon-arrow-bbf/src/datafusion/table_function.rs index c209cde9..e71c7168 100644 --- a/beacon-file-formats/beacon-arrow-bbf/src/datafusion/table_function.rs +++ b/beacon-file-formats/beacon-arrow-bbf/src/datafusion/table_function.rs @@ -4,10 +4,8 @@ use std::sync::Arc; use arrow::datatypes::{DataType, Field}; use beacon_common::{listing_url::parse_listing_table_url, super_table::SuperListingTable}; use datafusion::{ - catalog::TableFunctionImpl, - datasource::file_format::FileFormatFactory, - execution::object_store::ObjectStoreUrl, - prelude::SessionContext, + catalog::TableFunctionImpl, datasource::file_format::FileFormatFactory, + execution::object_store::ObjectStoreUrl, prelude::SessionContext, }; use beacon_common::table_function::BeaconTableFunctionImpl; diff --git a/beacon-file-formats/beacon-arrow-csv/src/datafusion/mod.rs b/beacon-file-formats/beacon-arrow-csv/src/datafusion/mod.rs index 0ea4358a..9f70fa14 100644 --- a/beacon-file-formats/beacon-arrow-csv/src/datafusion/mod.rs +++ b/beacon-file-formats/beacon-arrow-csv/src/datafusion/mod.rs @@ -170,7 +170,10 @@ impl FileFormat for CsvFormat { .await } - fn file_source(&self, table_schema: datafusion::datasource::table_schema::TableSchema) -> Arc { + fn file_source( + &self, + table_schema: datafusion::datasource::table_schema::TableSchema, + ) -> Arc { self.inner_format.file_source(table_schema) } } diff --git a/beacon-file-formats/beacon-arrow-csv/src/datafusion/table_function.rs b/beacon-file-formats/beacon-arrow-csv/src/datafusion/table_function.rs index 990de461..114857ad 100644 --- a/beacon-file-formats/beacon-arrow-csv/src/datafusion/table_function.rs +++ b/beacon-file-formats/beacon-arrow-csv/src/datafusion/table_function.rs @@ -1,8 +1,8 @@ use std::sync::Arc; +use crate::datafusion::CsvFormat; use arrow::datatypes::{DataType, Field}; use beacon_common::{listing_url::parse_listing_table_url, super_table::SuperListingTable}; -use crate::datafusion::CsvFormat; use datafusion::{ catalog::TableFunctionImpl, common::plan_err, diff --git a/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/mod.rs b/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/mod.rs index acfd1438..ac6674aa 100644 --- a/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/mod.rs +++ b/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/mod.rs @@ -432,8 +432,14 @@ mod tests { // Geometry column round-trips as a native GeoArrow struct (x, y children). let geom_idx = full.schema().index_of("geometry").expect("geometry column"); let geom = full.column(geom_idx).as_struct(); - let x = geom.column_by_name("x").expect("x child").as_primitive::(); - let y = geom.column_by_name("y").expect("y child").as_primitive::(); + let x = geom + .column_by_name("x") + .expect("x child") + .as_primitive::(); + let y = geom + .column_by_name("y") + .expect("y child") + .as_primitive::(); assert_eq!(x.values(), &[1.0, 3.0, 5.0]); assert_eq!(y.values(), &[2.0, 4.0, 6.0]); } @@ -467,7 +473,11 @@ mod tests { let full = arrow::compute::concat_batches(&batches[0].schema(), &batches).expect("concat"); let out_schema = full.schema(); - let names: Vec<&str> = out_schema.fields().iter().map(|f| f.name().as_str()).collect(); + let names: Vec<&str> = out_schema + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect(); assert_eq!(names, vec!["id"], "only the projected column should remain"); assert_eq!(full.num_rows(), 3); assert_eq!( @@ -524,8 +534,7 @@ mod tests { assert!(inferred.field_with_name("id").is_ok()); assert!(inferred.field_with_name("geometry").is_err()); - let opener = - opener::GeoParquetOpener::new(object_store, inferred.clone(), 128 * 1024); + let opener = opener::GeoParquetOpener::new(object_store, inferred.clone(), 128 * 1024); let stream = opener .open(PartitionedFile::from(object)) .expect("open") diff --git a/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/reader.rs b/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/reader.rs index ee62ba41..ffb675ac 100644 --- a/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/reader.rs +++ b/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/reader.rs @@ -29,8 +29,8 @@ pub(crate) async fn stream_builder( object_store: Arc, object: &ObjectMeta, ) -> Result { - let reader = ParquetObjectReader::new(object_store, object.location.clone()) - .with_file_size(object.size); + let reader = + ParquetObjectReader::new(object_store, object.location.clone()).with_file_size(object.size); ParquetRecordBatchStreamBuilder::new(reader) .await diff --git a/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/table_function.rs b/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/table_function.rs index 9613724e..e7400a3a 100644 --- a/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/table_function.rs +++ b/beacon-file-formats/beacon-arrow-geoparquet/src/datafusion/table_function.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use arrow::datatypes::{DataType, Field}; use crate::datafusion::{GeoParquetFormat, GeoParquetOptions}; +use arrow::datatypes::{DataType, Field}; use beacon_common::{listing_url::parse_listing_table_url, super_table::SuperListingTable}; use datafusion::{ catalog::TableFunctionImpl, execution::object_store::ObjectStoreUrl, prelude::SessionContext, @@ -66,7 +66,8 @@ impl TableFunctionImpl for ReadGeoParquetFunc { &self, args: &[datafusion::prelude::Expr], ) -> datafusion::error::Result> { - let glob_paths = beacon_common::table_function::parse_glob_paths_arg(args, "read_geoparquet")?; + let glob_paths = + beacon_common::table_function::parse_glob_paths_arg(args, "read_geoparquet")?; tracing::debug!("read_geoparquet glob paths: {:?}", glob_paths); diff --git a/beacon-file-formats/beacon-arrow-ipc/src/datafusion/mod.rs b/beacon-file-formats/beacon-arrow-ipc/src/datafusion/mod.rs index 2afbc11f..0d7f11bc 100644 --- a/beacon-file-formats/beacon-arrow-ipc/src/datafusion/mod.rs +++ b/beacon-file-formats/beacon-arrow-ipc/src/datafusion/mod.rs @@ -174,7 +174,10 @@ impl FileFormat for ArrowFormat { .await } - fn file_source(&self, table_schema: datafusion::datasource::table_schema::TableSchema) -> Arc { + fn file_source( + &self, + table_schema: datafusion::datasource::table_schema::TableSchema, + ) -> Arc { self.inner_format.file_source(table_schema) } } diff --git a/beacon-file-formats/beacon-arrow-ipc/src/datafusion/table_function.rs b/beacon-file-formats/beacon-arrow-ipc/src/datafusion/table_function.rs index 0127d2b7..202232dd 100644 --- a/beacon-file-formats/beacon-arrow-ipc/src/datafusion/table_function.rs +++ b/beacon-file-formats/beacon-arrow-ipc/src/datafusion/table_function.rs @@ -1,8 +1,8 @@ use std::sync::Arc; +use crate::datafusion::ArrowFormat; use arrow::datatypes::{DataType, Field}; use beacon_common::{listing_url::parse_listing_table_url, super_table::SuperListingTable}; -use crate::datafusion::ArrowFormat; use datafusion::{ catalog::TableFunctionImpl, execution::object_store::ObjectStoreUrl, prelude::SessionContext, }; diff --git a/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/mod.rs b/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/mod.rs index 7904df5d..fbb9607c 100644 --- a/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/mod.rs +++ b/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/mod.rs @@ -602,7 +602,9 @@ mod tests { options.insert("use_reader_cache".to_string(), "false".to_string()); options.insert("enable_statistics".to_string(), "false".to_string()); options.insert("read_dimensions".to_string(), "time, lat".to_string()); - let format = factory.create(&ctx.state(), &options).expect("valid options"); + let format = factory + .create(&ctx.state(), &options) + .expect("valid options"); let netcdf = format .as_any() .downcast_ref::() @@ -785,7 +787,8 @@ mod tests { let missing_idx = merged.index_of(&missing).unwrap(); // No projection pushed → the opener reads under the full merged schema. - let ts = datafusion::datasource::table_schema::TableSchema::from_file_schema(merged.clone()); + let ts = + datafusion::datasource::table_schema::TableSchema::from_file_schema(merged.clone()); let opener = source::NetCDFSource::new(store, None, ts); let conf = FileScanConfigBuilder::new( ObjectStoreUrl::local_filesystem(), @@ -841,9 +844,8 @@ mod tests { .await .expect("dim schema"); - let ts = datafusion::datasource::table_schema::TableSchema::from_file_schema( - dim_schema.clone(), - ); + let ts = + datafusion::datasource::table_schema::TableSchema::from_file_schema(dim_schema.clone()); let opener = source::NetCDFSource::new(store, Some(vec!["time".to_string()]), ts); let file_opener = { let conf = FileScanConfigBuilder::new( @@ -944,7 +946,10 @@ mod tests { .iter() .map(|b| b.num_rows()) .sum(); - assert_eq!(rows, 0, "impossible latitude predicate should yield no rows"); + assert_eq!( + rows, 0, + "impossible latitude predicate should yield no rows" + ); } #[tokio::test] @@ -997,13 +1002,12 @@ mod tests { .unwrap(); let mut kept = 0i64; for b in &batches { - let col = b - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); + let col = b.column(0).as_any().downcast_ref::().unwrap(); for i in 0..col.len() { - assert!(col.value(i) > mid, "every returned lat must satisfy the predicate"); + assert!( + col.value(i) > mid, + "every returned lat must satisfy the predicate" + ); } kept += b.num_rows() as i64; } diff --git a/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/sink.rs b/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/sink.rs index b04697a0..a8a9b078 100644 --- a/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/sink.rs +++ b/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/sink.rs @@ -1092,8 +1092,7 @@ mod tests { insert_op: InsertOp::Append, keep_partition_by_columns: false, file_extension: "nc".to_string(), - file_output_mode: - datafusion::datasource::physical_plan::FileOutputMode::SingleFile, + file_output_mode: datafusion::datasource::physical_plan::FileOutputMode::SingleFile, } } diff --git a/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/source.rs b/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/source.rs index 0fad4855..5409b23a 100644 --- a/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/source.rs +++ b/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/source.rs @@ -6,8 +6,7 @@ use arrow::{ }; use beacon_nd_array::{ arrow::{ - batch::any_dataset_as_record_batch_stream, - metrics::DatasetReadMetrics, + batch::any_dataset_as_record_batch_stream, metrics::DatasetReadMetrics, pushdown_filter::PushdownFilter, }, projection::DatasetProjection, @@ -385,21 +384,22 @@ impl NetCDFOpener { }; let pushdown_filter = predicate.map(PushdownFilter::new); - let stream = any_dataset_as_record_batch_stream(dataset, batch_size, pushdown_filter, metrics) - .map_err(|e| { - datafusion::error::DataFusionError::Execution(format!( - "Error reading NetCDF as Arrow stream: {e}" - )) - }) - .and_then(move |batch| { - let mapped = adapter.adapt_batch(&batch).map_err(|e| { + let stream = + any_dataset_as_record_batch_stream(dataset, batch_size, pushdown_filter, metrics) + .map_err(|e| { datafusion::error::DataFusionError::Execution(format!( - "Failed to adapt NetCDF batch schema: {e}" + "Error reading NetCDF as Arrow stream: {e}" )) - }); - futures::future::ready(mapped) - }) - .boxed(); + }) + .and_then(move |batch| { + let mapped = adapter.adapt_batch(&batch).map_err(|e| { + datafusion::error::DataFusionError::Execution(format!( + "Failed to adapt NetCDF batch schema: {e}" + )) + }); + futures::future::ready(mapped) + }) + .boxed(); Ok(stream) } diff --git a/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/table_function.rs b/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/table_function.rs index 5cda3d4f..888b5f5e 100644 --- a/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/table_function.rs +++ b/beacon-file-formats/beacon-arrow-netcdf/src/datafusion/table_function.rs @@ -114,11 +114,14 @@ impl TableFunctionImpl for ReadNetCDFFunc { } let state = self.session_ctx.state(); - let factory = state.get_file_format_factory(NETCDF_FORMAT).ok_or_else(|| { - datafusion::error::DataFusionError::Execution( - "read_netcdf: the NetCDF file format is not registered on the session".to_string(), - ) - })?; + let factory = state + .get_file_format_factory(NETCDF_FORMAT) + .ok_or_else(|| { + datafusion::error::DataFusionError::Execution( + "read_netcdf: the NetCDF file format is not registered on the session" + .to_string(), + ) + })?; let file_format = factory.create(&state, &format_options)?; let super_listing_table = tokio::task::block_in_place(|| { diff --git a/beacon-file-formats/beacon-arrow-netcdf/src/encoders/default.rs b/beacon-file-formats/beacon-arrow-netcdf/src/encoders/default.rs index 0bb36173..0d8a18c9 100644 --- a/beacon-file-formats/beacon-arrow-netcdf/src/encoders/default.rs +++ b/beacon-file-formats/beacon-arrow-netcdf/src/encoders/default.rs @@ -158,7 +158,9 @@ impl DefaultEncoder { .as_any() .downcast_ref::() .ok_or_else(|| { - anyhow::anyhow!("failed to downcast column {var_name} to FixedSizeBinaryArray") + anyhow::anyhow!( + "failed to downcast column {var_name} to FixedSizeBinaryArray" + ) })?; extents.push(0..*size as usize); @@ -176,7 +178,9 @@ impl DefaultEncoder { })?; variable.put(view, extents).map_err(|e| { - anyhow::anyhow!("failed to write FixedSizeBinary column {var_name} to NetCDF: {e}") + anyhow::anyhow!( + "failed to write FixedSizeBinary column {var_name} to NetCDF: {e}" + ) })?; } DataType::Utf8 => { diff --git a/beacon-file-formats/beacon-arrow-netcdf/src/reader.rs b/beacon-file-formats/beacon-arrow-netcdf/src/reader.rs index 01a0ec4d..09539f27 100644 --- a/beacon-file-formats/beacon-arrow-netcdf/src/reader.rs +++ b/beacon-file-formats/beacon-arrow-netcdf/src/reader.rs @@ -1,4 +1,4 @@ -//! High-level NetCDF reader that produces [`AnyDataset`] values. +//! High-level NetCDF reader that produces [`AnyDataset`] values. //! //! The entry point is [`open_dataset`], which opens a NetCDF file, converts //! every variable and attribute into a lazy [`NdArrayD`] wrapper, and returns diff --git a/beacon-file-formats/beacon-arrow-odv/src/datafusion/source.rs b/beacon-file-formats/beacon-arrow-odv/src/datafusion/source.rs index fb2436cc..9aac5955 100644 --- a/beacon-file-formats/beacon-arrow-odv/src/datafusion/source.rs +++ b/beacon-file-formats/beacon-arrow-odv/src/datafusion/source.rs @@ -13,7 +13,7 @@ use datafusion::{ schema_adapter::SchemaAdapterFactory, table_schema::TableSchema, }, - physical_expr::{LexOrdering, projection::ProjectionExprs}, + physical_expr::{projection::ProjectionExprs, LexOrdering}, physical_expr_adapter::BatchAdapterFactory, physical_plan::metrics::ExecutionPlanMetricsSet, }; diff --git a/beacon-file-formats/beacon-arrow-odv/src/datafusion/table_function.rs b/beacon-file-formats/beacon-arrow-odv/src/datafusion/table_function.rs index db9fac91..b67447e9 100644 --- a/beacon-file-formats/beacon-arrow-odv/src/datafusion/table_function.rs +++ b/beacon-file-formats/beacon-arrow-odv/src/datafusion/table_function.rs @@ -1,8 +1,8 @@ use std::sync::Arc; +use crate::datafusion::OdvFormat; use arrow::datatypes::{DataType, Field}; use beacon_common::{listing_url::parse_listing_table_url, super_table::SuperListingTable}; -use crate::datafusion::OdvFormat; use datafusion::{ catalog::TableFunctionImpl, execution::object_store::ObjectStoreUrl, @@ -64,7 +64,8 @@ impl TableFunctionImpl for ReadOdvAsciiFunc { &self, args: &[Expr], ) -> datafusion::error::Result> { - let glob_paths = beacon_common::table_function::parse_glob_paths_arg(args, "read_odv_ascii")?; + let glob_paths = + beacon_common::table_function::parse_glob_paths_arg(args, "read_odv_ascii")?; tracing::debug!("read_odv_ascii glob paths: {:?}", glob_paths); diff --git a/beacon-file-formats/beacon-arrow-odv/src/reader.rs b/beacon-file-formats/beacon-arrow-odv/src/reader.rs index 5e261f83..bc9853ab 100644 --- a/beacon-file-formats/beacon-arrow-odv/src/reader.rs +++ b/beacon-file-formats/beacon-arrow-odv/src/reader.rs @@ -491,9 +491,18 @@ mod unit_tests { // Units in brackets are stripped from the name and stored as metadata. assert_eq!(field.name(), "Longitude"); assert_eq!(field.data_type(), &DataType::Float32); - assert_eq!(field.metadata().get("units").map(|s| s.as_str()), Some("degrees east")); - assert_eq!(field.metadata().get("qf_schema").map(|s| s.as_str()), Some("SEADATANET")); - assert_eq!(field.metadata().get("comment").map(|s| s.as_str()), Some("pos")); + assert_eq!( + field.metadata().get("units").map(|s| s.as_str()), + Some("degrees east") + ); + assert_eq!( + field.metadata().get("qf_schema").map(|s| s.as_str()), + Some("SEADATANET") + ); + assert_eq!( + field.metadata().get("comment").map(|s| s.as_str()), + Some("pos") + ); } #[test] @@ -529,8 +538,10 @@ mod unit_tests { let batch = RecordBatch::try_new( input.clone(), - vec![Arc::new(arrow::array::Float32Array::from(vec![1.0_f32, 2.0])) - as Arc], + vec![ + Arc::new(arrow::array::Float32Array::from(vec![1.0_f32, 2.0])) + as Arc, + ], ) .unwrap(); let mapped = mapper.map_batch(batch, None).unwrap(); diff --git a/beacon-file-formats/beacon-arrow-parquet/src/datafusion/mod.rs b/beacon-file-formats/beacon-arrow-parquet/src/datafusion/mod.rs index c14dbc39..22e325e5 100644 --- a/beacon-file-formats/beacon-arrow-parquet/src/datafusion/mod.rs +++ b/beacon-file-formats/beacon-arrow-parquet/src/datafusion/mod.rs @@ -177,7 +177,10 @@ impl FileFormat for ParquetFormat { .await } - fn file_source(&self, table_schema: datafusion::datasource::table_schema::TableSchema) -> Arc { + fn file_source( + &self, + table_schema: datafusion::datasource::table_schema::TableSchema, + ) -> Arc { self.inner.file_source(table_schema) } } @@ -202,8 +205,10 @@ fn cast_ts_seconds_to_ms( let expr = cast(Expr::Column(Column::new_unqualified(&name)), target); session.create_physical_expr(expr, &df_schema)? } - _ => session - .create_physical_expr(Expr::Column(Column::new_unqualified(&name)), &df_schema)?, + _ => session.create_physical_expr( + Expr::Column(Column::new_unqualified(&name)), + &df_schema, + )?, }; Ok((expr, name)) }) diff --git a/beacon-file-formats/beacon-arrow-parquet/src/datafusion/table_function.rs b/beacon-file-formats/beacon-arrow-parquet/src/datafusion/table_function.rs index 548bdd4f..97dc44c8 100644 --- a/beacon-file-formats/beacon-arrow-parquet/src/datafusion/table_function.rs +++ b/beacon-file-formats/beacon-arrow-parquet/src/datafusion/table_function.rs @@ -1,8 +1,8 @@ use std::sync::Arc; +use crate::datafusion::ParquetFormat; use arrow::datatypes::{DataType, Field}; use beacon_common::{listing_url::parse_listing_table_url, super_table::SuperListingTable}; -use crate::datafusion::ParquetFormat; use datafusion::{ catalog::TableFunctionImpl, execution::object_store::ObjectStoreUrl, prelude::SessionContext, }; diff --git a/beacon-file-formats/beacon-arrow-tiff/src/datafusion/mod.rs b/beacon-file-formats/beacon-arrow-tiff/src/datafusion/mod.rs index cc8c2141..5198dbfa 100644 --- a/beacon-file-formats/beacon-arrow-tiff/src/datafusion/mod.rs +++ b/beacon-file-formats/beacon-arrow-tiff/src/datafusion/mod.rs @@ -189,9 +189,9 @@ mod tests { use datafusion::datasource::physical_plan::{FileScanConfigBuilder, FileSource}; use datafusion::execution::object_store::ObjectStoreUrl; use futures::StreamExt; + use object_store::ObjectStoreExt; use object_store::memory::InMemory; use object_store::path::Path; - use object_store::ObjectStoreExt; const TEST_TIF_BYTES: &[u8] = include_bytes!("../../test-files/test.tif"); @@ -262,7 +262,9 @@ mod tests { }; let stream = file_opener - .open(datafusion::datasource::listing::PartitionedFile::from(object)) + .open(datafusion::datasource::listing::PartitionedFile::from( + object, + )) .expect("open") .await .expect("stream"); @@ -373,7 +375,9 @@ mod tests { }; let stream = file_opener - .open(datafusion::datasource::listing::PartitionedFile::from(object)) + .open(datafusion::datasource::listing::PartitionedFile::from( + object, + )) .expect("open") .await .expect("stream"); @@ -466,7 +470,10 @@ mod tests { .iter() .map(|b| b.num_rows()) .sum(); - assert_eq!(rows, 0, "impossible latitude predicate should yield no rows"); + assert_eq!( + rows, 0, + "impossible latitude predicate should yield no rows" + ); } #[tokio::test] @@ -490,7 +497,10 @@ mod tests { .downcast_ref::() .expect("geo.lat is Float64"); for i in 0..col.len() { - assert!(col.value(i) > 40.0, "every returned lat must satisfy the predicate"); + assert!( + col.value(i) > 40.0, + "every returned lat must satisfy the predicate" + ); } total += b.num_rows(); } diff --git a/beacon-file-formats/beacon-arrow-tiff/src/datafusion/source.rs b/beacon-file-formats/beacon-arrow-tiff/src/datafusion/source.rs index d28ee23b..1c16ecee 100644 --- a/beacon-file-formats/beacon-arrow-tiff/src/datafusion/source.rs +++ b/beacon-file-formats/beacon-arrow-tiff/src/datafusion/source.rs @@ -266,21 +266,22 @@ impl TiffOpener { }; let pushdown_filter = predicate.map(PushdownFilter::new); - let stream = any_dataset_as_record_batch_stream(dataset, batch_size, pushdown_filter, metrics) - .map_err(|e| { - datafusion::error::DataFusionError::Execution(format!( - "Error reading TIFF as Arrow stream: {e}" - )) - }) - .and_then(move |batch| { - let mapped = adapter.adapt_batch(&batch).map_err(|e| { + let stream = + any_dataset_as_record_batch_stream(dataset, batch_size, pushdown_filter, metrics) + .map_err(|e| { datafusion::error::DataFusionError::Execution(format!( - "Failed to adapt TIFF batch schema: {e}" + "Error reading TIFF as Arrow stream: {e}" )) - }); - futures::future::ready(mapped) - }) - .boxed(); + }) + .and_then(move |batch| { + let mapped = adapter.adapt_batch(&batch).map_err(|e| { + datafusion::error::DataFusionError::Execution(format!( + "Failed to adapt TIFF batch schema: {e}" + )) + }); + futures::future::ready(mapped) + }) + .boxed(); Ok(stream) } diff --git a/beacon-file-formats/beacon-arrow-tiff/src/datafusion/table_function.rs b/beacon-file-formats/beacon-arrow-tiff/src/datafusion/table_function.rs index 919f08dd..a6701ff9 100644 --- a/beacon-file-formats/beacon-arrow-tiff/src/datafusion/table_function.rs +++ b/beacon-file-formats/beacon-arrow-tiff/src/datafusion/table_function.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use arrow::datatypes::{DataType, Field}; use crate::datafusion::TiffFormat; +use arrow::datatypes::{DataType, Field}; use beacon_common::{listing_url::parse_listing_table_url, super_table::SuperListingTable}; use beacon_object_storage::DatasetsStore; use datafusion::{ diff --git a/beacon-file-formats/beacon-arrow-tiff/src/reader.rs b/beacon-file-formats/beacon-arrow-tiff/src/reader.rs index 05008072..6bd4be93 100644 --- a/beacon-file-formats/beacon-arrow-tiff/src/reader.rs +++ b/beacon-file-formats/beacon-arrow-tiff/src/reader.rs @@ -91,12 +91,12 @@ pub async fn open_dataset( } if let Some((tiles_x, tiles_y)) = first_ifd.tile_count() { - let tile_width = first_ifd - .tile_width() - .ok_or_else(|| anyhow::anyhow!("tiled TIFF reports a tile count but is missing TileWidth"))?; - let tile_height = first_ifd - .tile_height() - .ok_or_else(|| anyhow::anyhow!("tiled TIFF reports a tile count but is missing TileLength"))?; + let tile_width = first_ifd.tile_width().ok_or_else(|| { + anyhow::anyhow!("tiled TIFF reports a tile count but is missing TileWidth") + })?; + let tile_height = first_ifd.tile_height().ok_or_else(|| { + anyhow::anyhow!("tiled TIFF reports a tile count but is missing TileLength") + })?; insert_scalar(&mut arrays, "image.tile_width", tile_width)?; insert_scalar(&mut arrays, "image.tile_height", tile_height)?; insert_scalar(&mut arrays, "image.tile_count_x", tiles_x as u64)?; @@ -134,13 +134,15 @@ pub async fn open_dataset( )?; } - let nodata_value: Option = first_ifd.gdal_nodata().and_then(|s| match s.parse::() { - Ok(value) => Some(value), - Err(e) => { - tracing::warn!(value = %s, error = %e, "ignoring unparseable GDAL_NODATA tag"); - None - } - }); + let nodata_value: Option = first_ifd + .gdal_nodata() + .and_then(|s| match s.parse::() { + Ok(value) => Some(value), + Err(e) => { + tracing::warn!(value = %s, error = %e, "ignoring unparseable GDAL_NODATA tag"); + None + } + }); if let Some(nodata) = first_ifd.gdal_nodata() { insert_scalar(&mut arrays, "geo.nodata", nodata.to_string())?; @@ -429,7 +431,9 @@ async fn read_pixel_bands_stripped( None, ) .map_err(|e| { - anyhow::anyhow!("Failed to decompress strip at offset {offset} ({compression:?}): {e}") + anyhow::anyhow!( + "Failed to decompress strip at offset {offset} ({compression:?}): {e}" + ) })?; raw.extend_from_slice(&decoded); } @@ -627,7 +631,11 @@ mod tests { // The generator carves an 8×8 nodata block and fills the rest with the gradient. assert_eq!(n_nodata, 8 * 8, "unexpected nodata pixel count"); - assert_eq!(n_valid, WIDTH * HEIGHT - 8 * 8, "unexpected valid pixel count"); + assert_eq!( + n_valid, + WIDTH * HEIGHT - 8 * 8, + "unexpected valid pixel count" + ); // Row 0 starts inside the nodata block; the first valid pixel is at column 8. assert!(values[0] <= -1e30, "band.0[0] should be nodata"); diff --git a/beacon-file-formats/beacon-arrow-zarr/src/backend.rs b/beacon-file-formats/beacon-arrow-zarr/src/backend.rs index 9664284f..1aa6ee69 100644 --- a/beacon-file-formats/beacon-arrow-zarr/src/backend.rs +++ b/beacon-file-formats/beacon-arrow-zarr/src/backend.rs @@ -107,7 +107,10 @@ async fn read_raw_as_f64( ZarrDtypeKind::Float32 => read_widen!(f32), ZarrDtypeKind::Float64 => read_widen!(f64), other => { - anyhow::bail!("CF decoding is not supported for zarr dtype kind {:?}", other) + anyhow::bail!( + "CF decoding is not supported for zarr dtype kind {:?}", + other + ) } }; Ok(out) @@ -342,7 +345,10 @@ impl ArrayBackend for CfTimeBackend { self.fill_value } - async fn read_subset(&self, subset: ArraySubset) -> anyhow::Result> { + async fn read_subset( + &self, + subset: ArraySubset, + ) -> anyhow::Result> { let raw = read_raw_as_f64(&self.array, &to_zarr_subset(&subset), self.kind).await?; let epoch = self.epoch; let unit = self.unit; diff --git a/beacon-file-formats/beacon-arrow-zarr/src/compat.rs b/beacon-file-formats/beacon-arrow-zarr/src/compat.rs index 1386ebaf..4801da42 100644 --- a/beacon-file-formats/beacon-arrow-zarr/src/compat.rs +++ b/beacon-file-formats/beacon-arrow-zarr/src/compat.rs @@ -108,7 +108,14 @@ pub fn array_to_nd_array( && let Some((epoch, unit)) = parse_cf_time_units(units, calendar) { let backend = CfTimeBackend::new( - array, kind, shape, dimensions, chunk_shape, epoch, unit, raw_fill, + array, + kind, + shape, + dimensions, + chunk_shape, + epoch, + unit, + raw_fill, ); return Ok(Arc::new(NdArray::new_with_backend(backend)?)); } @@ -116,13 +123,8 @@ pub fn array_to_nd_array( // Direct read in the array's native dtype. macro_rules! direct { ($ty:ty, $fill:expr) => {{ - let backend = ZarrArrayBackend::<$ty>::new( - array, - shape, - dimensions, - chunk_shape, - $fill, - ); + let backend = + ZarrArrayBackend::<$ty>::new(array, shape, dimensions, chunk_shape, $fill); Ok(Arc::new(NdArray::new_with_backend(backend)?) as Arc) }}; } diff --git a/beacon-file-formats/beacon-arrow-zarr/src/datafusion/mod.rs b/beacon-file-formats/beacon-arrow-zarr/src/datafusion/mod.rs index e9f26f6d..01d62167 100644 --- a/beacon-file-formats/beacon-arrow-zarr/src/datafusion/mod.rs +++ b/beacon-file-formats/beacon-arrow-zarr/src/datafusion/mod.rs @@ -479,7 +479,10 @@ mod tests { .iter() .map(|f| f.name().clone()) .collect(); - assert!(names.contains(&"time".to_string()), "time present: {names:?}"); + assert!( + names.contains(&"time".to_string()), + "time present: {names:?}" + ); assert!( !names.contains(&"analysed_sst".to_string()), "analysed_sst depends on lat/lon and must be excluded: {names:?}" @@ -603,13 +606,12 @@ mod tests { .unwrap(); let mut kept = 0i64; for b in &batches { - let col = b - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); + let col = b.column(0).as_any().downcast_ref::().unwrap(); for i in 0..col.len() { - assert!(col.value(i) > mid, "every returned lat must satisfy the predicate"); + assert!( + col.value(i) > mid, + "every returned lat must satisfy the predicate" + ); } kept += b.num_rows() as i64; } diff --git a/beacon-file-formats/beacon-arrow-zarr/src/datafusion/source.rs b/beacon-file-formats/beacon-arrow-zarr/src/datafusion/source.rs index 00abb833..33af3761 100644 --- a/beacon-file-formats/beacon-arrow-zarr/src/datafusion/source.rs +++ b/beacon-file-formats/beacon-arrow-zarr/src/datafusion/source.rs @@ -177,13 +177,11 @@ impl FileSource for ZarrSource { ..self.clone() }; - Ok( - FilterPushdownPropagation::with_parent_pushdown_result(vec![ - PushedDown::No; - filters.len() - ]) - .with_updated_node(Arc::new(source)), - ) + Ok(FilterPushdownPropagation::with_parent_pushdown_result(vec![ + PushedDown::No; + filters.len() + ]) + .with_updated_node(Arc::new(source))) } } @@ -202,7 +200,9 @@ struct ZarrOpener { impl FileOpener for ZarrOpener { fn open(&self, file: PartitionedFile) -> datafusion::error::Result { let zarr_path = ZarrPath::new_from_object_meta(file.object_meta.clone()).map_err(|e| { - DataFusionError::Execution(format!("Failed to create ZarrPath from object metadata: {e}")) + DataFusionError::Execution(format!( + "Failed to create ZarrPath from object metadata: {e}" + )) })?; let object_store = self.object_store.clone(); @@ -245,9 +245,10 @@ impl FileOpener for ZarrOpener { None => full, }; - let file_schema: SchemaRef = Arc::new(any_dataset_to_arrow_schema(&full).map_err( - |e| DataFusionError::Execution(format!("Failed to derive Zarr Arrow schema: {e}")), - )?); + let file_schema: SchemaRef = + Arc::new(any_dataset_to_arrow_schema(&full).map_err(|e| { + DataFusionError::Execution(format!("Failed to derive Zarr Arrow schema: {e}")) + })?); // Columns of this group that the query needs, in file order — used // both to prune the read and as the source schema for the adapter. @@ -332,22 +333,22 @@ impl FileOpener for ZarrOpener { })?; let pushdown_filter = predicate.map(PushdownFilter::new); - let stream = any_dataset_as_record_batch_stream( - projected, - batch_size, - pushdown_filter, - metrics, - ) - .map_err(|e| { - DataFusionError::Execution(format!("Error reading Zarr dataset as Arrow: {e}")) - }) - .and_then(move |batch| { - let mapped = adapter.adapt_batch(&batch).map_err(|e| { - DataFusionError::Execution(format!("Failed to adapt Zarr batch schema: {e}")) - }); - future::ready(mapped) - }) - .boxed(); + let stream = + any_dataset_as_record_batch_stream(projected, batch_size, pushdown_filter, metrics) + .map_err(|e| { + DataFusionError::Execution(format!( + "Error reading Zarr dataset as Arrow: {e}" + )) + }) + .and_then(move |batch| { + let mapped = adapter.adapt_batch(&batch).map_err(|e| { + DataFusionError::Execution(format!( + "Failed to adapt Zarr batch schema: {e}" + )) + }); + future::ready(mapped) + }) + .boxed(); Ok(stream) }; diff --git a/beacon-file-formats/beacon-arrow-zarr/src/datafusion/table_function.rs b/beacon-file-formats/beacon-arrow-zarr/src/datafusion/table_function.rs index 7b082d72..6171c81a 100644 --- a/beacon-file-formats/beacon-arrow-zarr/src/datafusion/table_function.rs +++ b/beacon-file-formats/beacon-arrow-zarr/src/datafusion/table_function.rs @@ -1,8 +1,8 @@ use std::{fmt::Debug, sync::Arc}; +use crate::datafusion::ZarrFormat; use arrow::datatypes::{DataType, Field}; use beacon_common::{listing_url::parse_listing_table_url, super_table::SuperListingTable}; -use crate::datafusion::ZarrFormat; use datafusion::{ catalog::TableFunctionImpl, common::plan_err, diff --git a/beacon-file-formats/beacon-arrow-zarr/src/reader.rs b/beacon-file-formats/beacon-arrow-zarr/src/reader.rs index 139685d5..f67b9471 100644 --- a/beacon-file-formats/beacon-arrow-zarr/src/reader.rs +++ b/beacon-file-formats/beacon-arrow-zarr/src/reader.rs @@ -66,7 +66,10 @@ pub async fn dataset_from_group( let array_name = array_node_path .strip_prefix(&group_path) .unwrap_or(&array_node_path); - let array_name = array_name.strip_prefix('/').unwrap_or(array_name).to_string(); + let array_name = array_name + .strip_prefix('/') + .unwrap_or(array_name) + .to_string(); // Parse the array's JSON attributes once: they drive both the // surfaced `{array}.{attr}` columns and CF decoding of the array. diff --git a/beacon-file-formats/beacon-delta/src/provider.rs b/beacon-file-formats/beacon-delta/src/provider.rs index 178cf60a..db56c374 100644 --- a/beacon-file-formats/beacon-delta/src/provider.rs +++ b/beacon-file-formats/beacon-delta/src/provider.rs @@ -239,7 +239,10 @@ fn location_to_prefix(location: &str) -> anyhow::Result { None => location, }; let trimmed = without_scheme.trim_matches('/'); - anyhow::ensure!(!trimmed.is_empty(), "Delta table location must not be empty"); + anyhow::ensure!( + !trimmed.is_empty(), + "Delta table location must not be empty" + ); Ok(trimmed.to_string()) } @@ -365,7 +368,10 @@ mod tests { #[test] fn location_to_prefix_strips_scheme_and_slashes() { - assert_eq!(location_to_prefix("datasets://argo/tbl").unwrap(), "argo/tbl"); + assert_eq!( + location_to_prefix("datasets://argo/tbl").unwrap(), + "argo/tbl" + ); assert_eq!(location_to_prefix("/argo/tbl/").unwrap(), "argo/tbl"); assert_eq!(location_to_prefix("argo/tbl").unwrap(), "argo/tbl"); assert!(location_to_prefix("datasets://").is_err()); @@ -390,7 +396,10 @@ mod tests { ); let mut opts = HashMap::new(); - opts.insert("format.timestamp".to_string(), "2026-01-01T00:00:00Z".to_string()); + opts.insert( + "format.timestamp".to_string(), + "2026-01-01T00:00:00Z".to_string(), + ); assert_eq!( TimeTravel::from_options(&opts).unwrap(), Some(TimeTravel::Timestamp("2026-01-01T00:00:00Z".to_string())) diff --git a/beacon-file-formats/beacon-delta/src/wrapper.rs b/beacon-file-formats/beacon-delta/src/wrapper.rs index 2880a200..c1d39195 100644 --- a/beacon-file-formats/beacon-delta/src/wrapper.rs +++ b/beacon-file-formats/beacon-delta/src/wrapper.rs @@ -12,6 +12,7 @@ use std::any::Any; use std::sync::Arc; use beacon_object_storage::DatasetsStore; +use datafusion::arrow::datatypes::SchemaRef; use datafusion::catalog::{Session, TableProvider}; use datafusion::common::{Constraints, Statistics}; use datafusion::datasource::TableType; @@ -20,7 +21,6 @@ use datafusion::logical_expr::dml::InsertOp; use datafusion::logical_expr::TableProviderFilterPushDown; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::Expr; -use datafusion::arrow::datatypes::SchemaRef; use crate::definition::DeltaTableDefinition; use crate::provider::{reopen_delta_provider, TimeTravel}; diff --git a/beacon-file-formats/beacon-iceberg/src/alter.rs b/beacon-file-formats/beacon-iceberg/src/alter.rs index 466efffb..e454225d 100644 --- a/beacon-file-formats/beacon-iceberg/src/alter.rs +++ b/beacon-file-formats/beacon-iceberg/src/alter.rs @@ -17,7 +17,7 @@ use arrow::datatypes::{DataType as ArrowDataType, Field, Schema as ArrowSchema}; use datafusion::catalog::TableProvider; use datafusion::common::ScalarValue; use datafusion::logical_expr::dml::InsertOp; -use datafusion::logical_expr::{cast, col, lit, Expr}; +use datafusion::logical_expr::{Expr, cast, col, lit}; use datafusion::prelude::SessionContext; use datafusion_iceberg::DataFusionTable; use iceberg_rust::catalog::Catalog; @@ -235,7 +235,10 @@ fn build_transform_exprs(original_names: &[String], changes: &[SchemaChange]) -> for change in changes { match change { SchemaChange::AddColumn { name, data_type } => { - entries.push((name.clone(), cast(lit(ScalarValue::Null), data_type.clone()))); + entries.push(( + name.clone(), + cast(lit(ScalarValue::Null), data_type.clone()), + )); } SchemaChange::DropColumn { name } => { entries.retain(|(out_name, _)| out_name != name); diff --git a/beacon-file-formats/beacon-iceberg/src/catalog.rs b/beacon-file-formats/beacon-iceberg/src/catalog.rs index 26ed6d30..3a45fe3a 100644 --- a/beacon-file-formats/beacon-iceberg/src/catalog.rs +++ b/beacon-file-formats/beacon-iceberg/src/catalog.rs @@ -13,9 +13,9 @@ use beacon_object_storage::DATASETS_WRITEABLE_PREFIX; use iceberg_file_catalog::FileCatalog; use iceberg_rust::catalog::Catalog; use iceberg_rust::object_store::ObjectStoreBuilder; +use object_store::ObjectStore; use object_store::local::LocalFileSystem; use object_store::prefix::PrefixStore; -use object_store::ObjectStore; use tokio::sync::OnceCell; /// The single Iceberg namespace beacon creates managed tables under. @@ -70,7 +70,10 @@ pub async fn init_datasets_warehouse( // endpoint/region explicitly keeps the Iceberg warehouse on the same // backend as the datasets without re-reading the environment. let mut builder = ObjectStoreBuilder::s3() - .with_config("aws_allow_http", if s3.allow_http { "true" } else { "false" }) + .with_config( + "aws_allow_http", + if s3.allow_http { "true" } else { "false" }, + ) .and_then(|builder| { builder.with_config( "aws_virtual_hosted_style_request", @@ -85,12 +88,14 @@ pub async fn init_datasets_warehouse( if let Some(endpoint) = &s3.endpoint { builder = builder .with_config("aws_endpoint", endpoint) - .map_err(|error| anyhow::anyhow!("Failed to configure Iceberg S3 store: {error}"))?; + .map_err(|error| { + anyhow::anyhow!("Failed to configure Iceberg S3 store: {error}") + })?; } if let Some(region) = &s3.region { - builder = builder - .with_config("aws_region", region) - .map_err(|error| anyhow::anyhow!("Failed to configure Iceberg S3 store: {error}"))?; + builder = builder.with_config("aws_region", region).map_err(|error| { + anyhow::anyhow!("Failed to configure Iceberg S3 store: {error}") + })?; } (builder, format!("s3://{}/{warehouse_prefix}", s3.bucket)) } else { @@ -113,8 +118,10 @@ pub async fn init_datasets_warehouse( // maps paths under `__beacon__`), nested one level into the `iceberg` // sub-directory so a table's `/
` prefix resolves to // `__beacon__/iceberg//
`. - let drop_store: Arc = - Arc::new(PrefixStore::new(datasets.internal_store(), WAREHOUSE_SUBDIR)); + let drop_store: Arc = Arc::new(PrefixStore::new( + datasets.internal_store(), + WAREHOUSE_SUBDIR, + )); init_catalog(catalog); let _ = WAREHOUSE_STORE.set(drop_store); diff --git a/beacon-file-formats/beacon-iceberg/src/definition.rs b/beacon-file-formats/beacon-iceberg/src/definition.rs index ce5386d7..5dce487b 100644 --- a/beacon-file-formats/beacon-iceberg/src/definition.rs +++ b/beacon-file-formats/beacon-iceberg/src/definition.rs @@ -64,7 +64,10 @@ impl TableDefinition for IcebergTableDefinition { let table = match tabular { Tabular::Table(table) => table, - _ => anyhow::bail!("Iceberg identifier '{}' does not refer to a table", self.name), + _ => anyhow::bail!( + "Iceberg identifier '{}' does not refer to a table", + self.name + ), }; Ok(Arc::new(IcebergTable::new( diff --git a/beacon-file-formats/beacon-iceberg/src/lib.rs b/beacon-file-formats/beacon-iceberg/src/lib.rs index f5911e5d..a53600a2 100644 --- a/beacon-file-formats/beacon-iceberg/src/lib.rs +++ b/beacon-file-formats/beacon-iceberg/src/lib.rs @@ -20,18 +20,18 @@ use std::sync::Arc; use arrow::datatypes::Schema as ArrowSchema; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion_iceberg::table::write_parquet_data_files; use datafusion_iceberg::DataFusionTable; +use datafusion_iceberg::table::write_parquet_data_files; use futures::StreamExt; +use iceberg_rust::catalog::Catalog; use iceberg_rust::catalog::identifier::Identifier; use iceberg_rust::catalog::tabular::Tabular; -use iceberg_rust::catalog::Catalog; use iceberg_rust::table::Table; use object_store::{ObjectStore, ObjectStoreExt}; -pub use alter::{alter_table_schema, is_allowed_promotion, SchemaChange}; +pub use alter::{SchemaChange, alter_table_schema, is_allowed_promotion}; pub use catalog::{ - beacon_namespace, get_catalog, get_warehouse_store, init_catalog, BEACON_NAMESPACE, + BEACON_NAMESPACE, beacon_namespace, get_catalog, get_warehouse_store, init_catalog, }; pub use definition::IcebergTableDefinition; pub use provider::IcebergTable; @@ -84,7 +84,11 @@ pub async fn drop_iceberg_table( ); } - tracing::debug!(table = name, files = locations.len(), "deleting Iceberg table files"); + tracing::debug!( + table = name, + files = locations.len(), + "deleting Iceberg table files" + ); for location in locations { store .delete(&location) @@ -269,7 +273,10 @@ mod tests { .downcast_ref::() .unwrap() .value(0); - assert_eq!(count, 1, "row written before restart should survive discovery"); + assert_eq!( + count, 1, + "row written before restart should survive discovery" + ); } /// Count rows by loading a fresh provider straight from the catalog (the @@ -442,9 +449,13 @@ mod tests { .await .expect("add column should commit"); - let batches = - query_from_catalog(&catalog, &namespace, "orders", "SELECT count(age) AS c FROM orders") - .await; + let batches = query_from_catalog( + &catalog, + &namespace, + "orders", + "SELECT count(age) AS c FROM orders", + ) + .await; let non_null_age = batches[0] .column(0) .as_any() diff --git a/beacon-file-formats/beacon-lance/src/definition.rs b/beacon-file-formats/beacon-lance/src/definition.rs index 681581ce..cf42ff64 100644 --- a/beacon-file-formats/beacon-lance/src/definition.rs +++ b/beacon-file-formats/beacon-lance/src/definition.rs @@ -32,7 +32,11 @@ pub struct LanceTableDefinition { } impl LanceTableDefinition { - pub fn new(name: impl Into, namespace: Vec, location: impl Into) -> Self { + pub fn new( + name: impl Into, + namespace: Vec, + location: impl Into, + ) -> Self { Self { name: name.into(), namespace, diff --git a/beacon-file-formats/beacon-lance/src/index.rs b/beacon-file-formats/beacon-lance/src/index.rs index 7f2d55c4..3fd0ca93 100644 --- a/beacon-file-formats/beacon-lance/src/index.rs +++ b/beacon-file-formats/beacon-lance/src/index.rs @@ -8,9 +8,9 @@ use lance::dataset::builder::DatasetBuilder; use lance::index::DatasetIndexExt; -use lance_index::scalar::inverted::InvertedIndexParams; -use lance_index::scalar::ScalarIndexParams; use lance_index::IndexType; +use lance_index::scalar::ScalarIndexParams; +use lance_index::scalar::inverted::InvertedIndexParams; use crate::warehouse::LanceWarehouse; @@ -94,13 +94,25 @@ pub async fn create_index( ScalarIndexKind::Inverted => { let params = InvertedIndexParams::default(); dataset - .create_index(&[column], index_type, Some(name.to_string()), ¶ms, false) + .create_index( + &[column], + index_type, + Some(name.to_string()), + ¶ms, + false, + ) .await } ScalarIndexKind::BTree | ScalarIndexKind::Bitmap => { let params = ScalarIndexParams::default(); dataset - .create_index(&[column], index_type, Some(name.to_string()), ¶ms, false) + .create_index( + &[column], + index_type, + Some(name.to_string()), + ¶ms, + false, + ) .await } }; @@ -111,11 +123,7 @@ pub async fn create_index( } /// Drop the index named `name` from the Lance table at `uri`. -pub async fn drop_index( - warehouse: &LanceWarehouse, - uri: &str, - name: &str, -) -> anyhow::Result<()> { +pub async fn drop_index(warehouse: &LanceWarehouse, uri: &str, name: &str) -> anyhow::Result<()> { tracing::info!(uri = %uri, name, "dropping Lance index"); let lock = warehouse.lock(uri); @@ -135,10 +143,7 @@ pub async fn drop_index( } /// List the indexes on the Lance table at `uri` (name + indexed columns). -pub async fn list_indices( - warehouse: &LanceWarehouse, - uri: &str, -) -> anyhow::Result> { +pub async fn list_indices(warehouse: &LanceWarehouse, uri: &str) -> anyhow::Result> { let dataset = DatasetBuilder::from_uri(uri) .with_session(warehouse.session()) .load() diff --git a/beacon-file-formats/beacon-lance/src/io.rs b/beacon-file-formats/beacon-lance/src/io.rs index 13f39c64..0d4ff9f5 100644 --- a/beacon-file-formats/beacon-lance/src/io.rs +++ b/beacon-file-formats/beacon-lance/src/io.rs @@ -1,7 +1,7 @@ //! Low-level Lance dataset writes, shared by create / insert / replace. -use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use arrow::array::RecordBatch; use arrow::compute::cast; diff --git a/beacon-file-formats/beacon-lance/src/lib.rs b/beacon-file-formats/beacon-lance/src/lib.rs index 2bdd723f..2321b54c 100644 --- a/beacon-file-formats/beacon-lance/src/lib.rs +++ b/beacon-file-formats/beacon-lance/src/lib.rs @@ -30,13 +30,13 @@ use datafusion::execution::SendableRecordBatchStream; use futures::StreamExt; use object_store::{ObjectStore, ObjectStoreExt}; -pub use alter::{alter_table, SchemaChange}; +pub use alter::{SchemaChange, alter_table}; pub use definition::LanceTableDefinition; -pub use index::{create_index, drop_index, list_indices, IndexInfo, ScalarIndexKind}; +pub use index::{IndexInfo, ScalarIndexKind, create_index, drop_index, list_indices}; pub use io::WriteKind; pub use mutate::{delete_rows, update_rows}; pub use provider::LanceTable; -pub use warehouse::{beacon_namespace, LanceWarehouse, BEACON_NAMESPACE}; +pub use warehouse::{BEACON_NAMESPACE, LanceWarehouse, beacon_namespace}; /// Create a new, empty Lance table at the warehouse location for /// `namespace`/`name` and return a ready [`LanceTable`] provider. The empty @@ -175,7 +175,11 @@ mod tests { .collect() .await .unwrap(); - assert_eq!(count(&ctx, "orders").await, 3, "three inserted rows visible"); + assert_eq!( + count(&ctx, "orders").await, + 3, + "three inserted rows visible" + ); // DELETE WHERE id = 1 -> keep id <> 1. let keep = ctx @@ -188,7 +192,11 @@ mod tests { replace_table_contents(&warehouse, &location, keep) .await .expect("replace should succeed"); - assert_eq!(count(&ctx, "orders").await, 2, "one row removed, two survive"); + assert_eq!( + count(&ctx, "orders").await, + 2, + "one row removed, two survive" + ); } #[tokio::test] @@ -200,9 +208,14 @@ mod tests { let warehouse = test_warehouse(&dir); let namespace = beacon_namespace(); - let table = create_lance_table(warehouse.clone(), &namespace, "discovered", &sample_schema()) - .await - .unwrap(); + let table = create_lance_table( + warehouse.clone(), + &namespace, + "discovered", + &sample_schema(), + ) + .await + .unwrap(); let ctx = SessionContext::new(); ctx.register_table("discovered", Arc::new(table)).unwrap(); ctx.sql("INSERT INTO discovered VALUES (7, 'g')") @@ -214,8 +227,11 @@ mod tests { // Simulate restart: round-trip the definition through JSON and rebuild. let location = warehouse.table_uri(&namespace, "discovered"); - let definition: Arc = - Arc::new(LanceTableDefinition::new("discovered", namespace, location.clone())); + let definition: Arc = Arc::new(LanceTableDefinition::new( + "discovered", + namespace, + location.clone(), + )); let json = serde_json::to_string(&definition).unwrap(); assert!(json.contains("\"lance\""), "typetag tag present: {json}"); let restored: Arc = serde_json::from_str(&json).unwrap(); @@ -232,7 +248,11 @@ mod tests { let ctx2 = SessionContext::new(); ctx2.register_table("discovered", provider).unwrap(); - assert_eq!(count(&ctx2, "discovered").await, 1, "row survives discovery"); + assert_eq!( + count(&ctx2, "discovered").await, + 1, + "row survives discovery" + ); } #[tokio::test] @@ -340,11 +360,19 @@ mod tests { .await .unwrap(); - create_index(&warehouse, &location, "id", "id_idx", ScalarIndexKind::BTree) - .await - .expect("create index should succeed"); + create_index( + &warehouse, + &location, + "id", + "id_idx", + ScalarIndexKind::BTree, + ) + .await + .expect("create index should succeed"); - let listed = list_indices(&warehouse, &location).await.expect("list indices"); + let listed = list_indices(&warehouse, &location) + .await + .expect("list indices"); assert_eq!(listed.len(), 1, "one index present"); assert_eq!(listed[0].name, "id_idx"); assert_eq!(listed[0].columns, vec!["id".to_string()]); @@ -353,7 +381,10 @@ mod tests { .await .expect("drop index should succeed"); assert!( - list_indices(&warehouse, &location).await.unwrap().is_empty(), + list_indices(&warehouse, &location) + .await + .unwrap() + .is_empty(), "no indices after drop" ); } @@ -404,7 +435,11 @@ mod tests { .unwrap() .value(0); assert_eq!(name, "Z", "row 2 should be updated"); - assert_eq!(count(&ctx, "orders").await, 3, "UPDATE must not change row count"); + assert_eq!( + count(&ctx, "orders").await, + 3, + "UPDATE must not change row count" + ); // Native DELETE WHERE id = 1. delete_rows(&warehouse, &location, Some("id = 1")) @@ -416,6 +451,10 @@ mod tests { delete_rows(&warehouse, &location, None) .await .expect("delete-all should succeed"); - assert_eq!(count(&ctx, "orders").await, 0, "delete-all empties the table"); + assert_eq!( + count(&ctx, "orders").await, + 0, + "delete-all empties the table" + ); } } diff --git a/beacon-file-formats/beacon-lance/src/provider.rs b/beacon-file-formats/beacon-lance/src/provider.rs index 5e5e4ef5..4132145e 100644 --- a/beacon-file-formats/beacon-lance/src/provider.rs +++ b/beacon-file-formats/beacon-lance/src/provider.rs @@ -12,15 +12,15 @@ use std::sync::Arc; use arrow::datatypes::SchemaRef; use async_trait::async_trait; use datafusion::catalog::{Session, TableProvider}; -use datafusion::datasource::sink::DataSinkExec; use datafusion::datasource::TableType; +use datafusion::datasource::sink::DataSinkExec; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::logical_expr::dml::InsertOp; use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; -use lance::dataset::builder::DatasetBuilder; use lance::datafusion::LanceTableProvider; +use lance::dataset::builder::DatasetBuilder; use lance::session::Session as LanceSession; use crate::definition::LanceTableDefinition; @@ -57,7 +57,9 @@ impl LanceTable { ) -> anyhow::Result { let provider = open_read_provider(&definition.location, warehouse.session()) .await - .map_err(|e| anyhow::anyhow!("Failed to open Lance table '{}': {e}", definition.name))?; + .map_err(|e| { + anyhow::anyhow!("Failed to open Lance table '{}': {e}", definition.name) + })?; Ok(Self::new(definition, provider.schema(), warehouse)) } diff --git a/beacon-file-formats/beacon-lance/src/sink.rs b/beacon-file-formats/beacon-lance/src/sink.rs index a578a339..f09daefb 100644 --- a/beacon-file-formats/beacon-lance/src/sink.rs +++ b/beacon-file-formats/beacon-lance/src/sink.rs @@ -17,7 +17,7 @@ use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::metrics::MetricsSet; use datafusion::physical_plan::{DisplayAs, DisplayFormatType}; -use crate::io::{write_stream, WriteKind}; +use crate::io::{WriteKind, write_stream}; use crate::warehouse::LanceWarehouse; /// Sink that applies a stream of batches to a single Lance dataset (by URI). diff --git a/beacon-file-formats/beacon-lance/src/warehouse.rs b/beacon-file-formats/beacon-lance/src/warehouse.rs index 105a8f3e..2ad8cbe7 100644 --- a/beacon-file-formats/beacon-lance/src/warehouse.rs +++ b/beacon-file-formats/beacon-lance/src/warehouse.rs @@ -80,7 +80,12 @@ impl ObjectStoreProvider for BeaconTablesProvider { url: &Url, _storage_options: Option<&HashMap>, ) -> LanceResult { - Ok(format!("{}${}{}", url.scheme(), url.authority(), url.path())) + Ok(format!( + "{}${}{}", + url.scheme(), + url.authority(), + url.path() + )) } } diff --git a/beacon-file-formats/beacon-nd-array/src/arrow/array.rs b/beacon-file-formats/beacon-nd-array/src/arrow/array.rs index b11a2cf6..72b0d2f6 100644 --- a/beacon-file-formats/beacon-nd-array/src/arrow/array.rs +++ b/beacon-file-formats/beacon-nd-array/src/arrow/array.rs @@ -23,7 +23,10 @@ macro_rules! convert_ndarray { .as_any() .downcast_ref::>() .ok_or_else(|| { - tracing::error!(target_type = $label, "failed to downcast NdArray for Arrow conversion"); + tracing::error!( + target_type = $label, + "failed to downcast NdArray for Arrow conversion" + ); anyhow::anyhow!( "Failed to downcast NdArray to NdArray<{}> for Arrow conversion.", $label diff --git a/beacon-file-formats/beacon-nd-array/src/dataset/mod.rs b/beacon-file-formats/beacon-nd-array/src/dataset/mod.rs index 41d5af3f..32946baf 100644 --- a/beacon-file-formats/beacon-nd-array/src/dataset/mod.rs +++ b/beacon-file-formats/beacon-nd-array/src/dataset/mod.rs @@ -69,7 +69,11 @@ impl Dataset { let dimensions = self .dimensions .iter() - .filter(|(dim, _)| arrays.values().any(|array| array.dimensions().contains(dim))) + .filter(|(dim, _)| { + arrays + .values() + .any(|array| array.dimensions().contains(dim)) + }) .map(|(dim, size)| (dim.clone(), *size)) .collect(); @@ -111,12 +115,10 @@ impl Dataset { .map(|array| array.dimensions())?; // Already broadcast-safe: every variable's dims fit inside `max_dims`. - let needs_narrowing = self.arrays.values().any(|array| { - !array - .dimensions() - .iter() - .all(|dim| max_dims.contains(dim)) - }); + let needs_narrowing = self + .arrays + .values() + .any(|array| !array.dimensions().iter().all(|dim| max_dims.contains(dim))); if !needs_narrowing { return None; } @@ -1073,8 +1075,13 @@ mod tests { let len: usize = shape.iter().product(); let dim_names: Vec = dims.iter().map(|d| d.to_string()).collect(); Arc::new( - NdArray::::try_new_from_vec_in_mem(vec![0.0; len], shape.to_vec(), dim_names, None) - .unwrap(), + NdArray::::try_new_from_vec_in_mem( + vec![0.0; len], + shape.to_vec(), + dim_names, + None, + ) + .unwrap(), ) } @@ -1083,7 +1090,10 @@ mod tests { // 2D var plus a 1D subset — already broadcast-safe. let ds = make_dataset( "safe", - vec![("grid", arr(&["x", "y"]).await), ("scale", arr(&["y"]).await)], + vec![ + ("grid", arr(&["x", "y"]).await), + ("scale", arr(&["y"]).await), + ], ) .await; assert_eq!(ds.default_broadcast_dimensions(), None); @@ -1221,7 +1231,10 @@ mod tests { // Equal variable count and equal dimensionality → first-encountered wins. let ds = make_dataset( "fulltie", - vec![("first", arr(&["a", "b"]).await), ("second", arr(&["c", "d"]).await)], + vec![ + ("first", arr(&["a", "b"]).await), + ("second", arr(&["c", "d"]).await), + ], ) .await; assert_eq!( diff --git a/beacon-file-formats/beacon-nd-array/src/dataset/ragged.rs b/beacon-file-formats/beacon-nd-array/src/dataset/ragged.rs index 479abc62..6628a0a7 100644 --- a/beacon-file-formats/beacon-nd-array/src/dataset/ragged.rs +++ b/beacon-file-formats/beacon-nd-array/src/dataset/ragged.rs @@ -315,7 +315,9 @@ impl RaggedDataset { } RaggedArray::ObservationVariable(array) => { let obs_dim = array.dimensions().first().cloned().ok_or_else(|| { - anyhow::anyhow!("observation variable {name} must have at least one dimension") + anyhow::anyhow!( + "observation variable {name} must have at least one dimension" + ) })?; let cum = &offsets[&obs_dim]; let obs_start = cum[index]; @@ -396,7 +398,9 @@ impl RaggedDataset { } RaggedArray::ObservationVariable(array) => { let obs_dim = array.dimensions().first().cloned().ok_or_else(|| { - anyhow::anyhow!("observation variable {name} must have at least one dimension") + anyhow::anyhow!( + "observation variable {name} must have at least one dimension" + ) })?; let cum = &offsets[&obs_dim]; let obs_start = cum[start]; diff --git a/beacon-file-formats/beacon-nd-array/src/error.rs b/beacon-file-formats/beacon-nd-array/src/error.rs index 2bc6283c..1cd8e851 100644 --- a/beacon-file-formats/beacon-nd-array/src/error.rs +++ b/beacon-file-formats/beacon-nd-array/src/error.rs @@ -28,7 +28,9 @@ pub enum NdArrayError { SizeMismatch { expected: usize, actual: usize }, /// A set of source dimensions is not a subset of the broadcast target. - #[error("source dimensions {source_dims:?} are not a subset of target dimensions {target_dims:?}")] + #[error( + "source dimensions {source_dims:?} are not a subset of target dimensions {target_dims:?}" + )] BroadcastDimensions { source_dims: Vec, target_dims: Vec, diff --git a/beacon-file-formats/beacon-nd-arrow/benches/broadcast.rs b/beacon-file-formats/beacon-nd-arrow/benches/broadcast.rs index 1ae72869..a1ee768f 100644 --- a/beacon-file-formats/beacon-nd-arrow/benches/broadcast.rs +++ b/beacon-file-formats/beacon-nd-arrow/benches/broadcast.rs @@ -104,9 +104,7 @@ fn bench_broadcast_scalar_to_nd(c: &mut Criterion) { group.bench_function(BenchmarkId::new("scalar_broadcast", case_name), |b| { b.iter(|| { rt.block_on(async { - black_box( - broadcast_and_build_batch(&arrays, schema.clone()).await, - ) + black_box(broadcast_and_build_batch(&arrays, schema.clone()).await) }) }); }); @@ -152,15 +150,21 @@ fn bench_broadcast_1d_to_nd(c: &mut Criterion) { for (case_name, target_shape, target_dims, n_out) in cases { group.throughput(Throughput::Elements(*n_out as u64)); - let schema = Arc::new(Schema::new(vec![Field::new("time", DataType::Int64, false)])); - let arrays = vec![(time_array.clone(), target_shape.clone(), target_dims.clone())]; + let schema = Arc::new(Schema::new(vec![Field::new( + "time", + DataType::Int64, + false, + )])); + let arrays = vec![( + time_array.clone(), + target_shape.clone(), + target_dims.clone(), + )]; group.bench_function(BenchmarkId::new("1d_broadcast", case_name), |b| { b.iter(|| { rt.block_on(async { - black_box( - broadcast_and_build_batch(&arrays, schema.clone()).await, - ) + black_box(broadcast_and_build_batch(&arrays, schema.clone()).await) }) }); }); @@ -223,9 +227,7 @@ fn bench_broadcast_2d_to_3d(c: &mut Criterion) { group.bench_function(BenchmarkId::new("2d_to_3d", case_name), |b| { b.iter(|| { rt.block_on(async { - black_box( - broadcast_and_build_batch(&arrays, schema.clone()).await, - ) + black_box(broadcast_and_build_batch(&arrays, schema.clone()).await) }) }); }); @@ -260,9 +262,7 @@ fn bench_broadcast_mixed(c: &mut Criterion) { vec![dstr("time")], ); // 2-D lat/lon grid - let lat_data: Vec = (0..n_2d) - .map(|i| -90.0 + (i / n_lon) as f64) - .collect(); + let lat_data: Vec = (0..n_2d).map(|i| -90.0 + (i / n_lon) as f64).collect(); let lat_array = make_f64( lat_data, vec![*n_lat, *n_lon], @@ -299,9 +299,7 @@ fn bench_broadcast_mixed(c: &mut Criterion) { group.bench_function(BenchmarkId::new("mixed_1d_2d_3d_scalar", case_name), |b| { b.iter(|| { rt.block_on(async { - black_box( - broadcast_and_build_batch(&arrays, schema.clone()).await, - ) + black_box(broadcast_and_build_batch(&arrays, schema.clone()).await) }) }); }); diff --git a/beacon-file-formats/beacon-nd-arrow/benches/convert.rs b/beacon-file-formats/beacon-nd-arrow/benches/convert.rs index a9c1dc93..130e584c 100644 --- a/beacon-file-formats/beacon-nd-arrow/benches/convert.rs +++ b/beacon-file-formats/beacon-nd-arrow/benches/convert.rs @@ -83,19 +83,27 @@ fn bench_convert_no_fill(c: &mut Criterion) { let i64_data: Vec = (0..n).map(|i| i as i64).collect(); group.bench_with_input(BenchmarkId::new("f64", n), &n, |b, _| { - b.iter(|| black_box(::arrow_from_array_view(&f64_data).unwrap())); + b.iter(|| { + black_box(::arrow_from_array_view(&f64_data).unwrap()) + }); }); group.bench_with_input(BenchmarkId::new("f32", n), &n, |b, _| { - b.iter(|| black_box(::arrow_from_array_view(&f32_data).unwrap())); + b.iter(|| { + black_box(::arrow_from_array_view(&f32_data).unwrap()) + }); }); group.bench_with_input(BenchmarkId::new("i32", n), &n, |b, _| { - b.iter(|| black_box(::arrow_from_array_view(&i32_data).unwrap())); + b.iter(|| { + black_box(::arrow_from_array_view(&i32_data).unwrap()) + }); }); group.bench_with_input(BenchmarkId::new("i64", n), &n, |b, _| { - b.iter(|| black_box(::arrow_from_array_view(&i64_data).unwrap())); + b.iter(|| { + black_box(::arrow_from_array_view(&i64_data).unwrap()) + }); }); } diff --git a/beacon-file-formats/beacon-nd-arrow/benches/flatten.rs b/beacon-file-formats/beacon-nd-arrow/benches/flatten.rs index 2eed1ae5..32ce9b09 100644 --- a/beacon-file-formats/beacon-nd-arrow/benches/flatten.rs +++ b/beacon-file-formats/beacon-nd-arrow/benches/flatten.rs @@ -8,8 +8,9 @@ use arrow::{ record_batch::RecordBatch, }; use beacon_nd_arrow::{ - NdRecordBatch, NdToArrowPipeOptions, pipe_nd_record_batch_stream, + NdRecordBatch, NdToArrowPipeOptions, array::{NdArrowArray, NdArrowArrayDispatch}, + pipe_nd_record_batch_stream, }; use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; use futures::StreamExt; @@ -74,8 +75,7 @@ fn bench_flatten_1d(c: &mut Criterion) { b.iter(|| { rt.block_on(async { let stream = batch.try_as_arrow_stream(cs).await.unwrap(); - let results: Vec> = - stream.collect().await; + let results: Vec> = stream.collect().await; black_box(results) }) }); @@ -150,8 +150,7 @@ fn bench_flatten_3d(c: &mut Criterion) { b.iter(|| { rt.block_on(async { let stream = batch.try_as_arrow_stream(cs).await.unwrap(); - let results: Vec> = - stream.collect().await; + let results: Vec> = stream.collect().await; black_box(results) }) }); @@ -194,12 +193,7 @@ fn bench_flatten_pipe_stream(c: &mut Criterion) { vec![dstr("time"), dstr("latitude"), dstr("longitude")], None, ), - arc_nd( - vec!["ship-A".to_string()], - vec![], - vec![], - None, - ), + arc_nd(vec!["ship-A".to_string()], vec![], vec![], None), ] }; @@ -234,8 +228,7 @@ fn bench_flatten_pipe_stream(c: &mut Criterion) { .collect(); let input_stream = futures::stream::iter(input_batches); let output = pipe_nd_record_batch_stream(input_stream, options); - let results: Vec> = - output.collect().await; + let results: Vec> = output.collect().await; black_box(results) }) }); diff --git a/beacon-file-formats/beacon-nd-arrow/benches/subset.rs b/beacon-file-formats/beacon-nd-arrow/benches/subset.rs index e8df960e..6c02459c 100644 --- a/beacon-file-formats/beacon-nd-arrow/benches/subset.rs +++ b/beacon-file-formats/beacon-nd-arrow/benches/subset.rs @@ -90,11 +90,7 @@ fn bench_subset_2d(c: &mut Criterion) { let n_lon = 2048usize; let n = n_lat * n_lon; let data: Vec = (0..n).map(|i| i as f64).collect(); - let array = make_f64( - data, - vec![n_lat, n_lon], - vec![dstr("lat"), dstr("lon")], - ); + let array = make_f64(data, vec![n_lat, n_lon], vec![dstr("lat"), dstr("lon")]); // (name, start, shape) let cases: &[(&str, Vec, Vec)] = &[ @@ -109,11 +105,7 @@ fn bench_subset_2d(c: &mut Criterion) { vec![256, 512], vec![512, 1024], // partial last axis → non-contiguous ), - ( - "small_inner_window", - vec![100, 100], - vec![64, 128], - ), + ("small_inner_window", vec![100, 100], vec![64, 128]), ]; let mut group = c.benchmark_group("subset_2d"); @@ -233,11 +225,7 @@ fn bench_subset_to_arrow_with_fill(c: &mut Criterion) { ) .unwrap(), ); - let array_no_fill = make_f64( - data, - vec![n_lat, n_lon], - vec![dstr("lat"), dstr("lon")], - ); + let array_no_fill = make_f64(data, vec![n_lat, n_lon], vec![dstr("lat"), dstr("lon")]); let start = vec![32, 32]; let shape = vec![128, 128]; diff --git a/beacon-file-formats/beacon-nd-arrow/src/array/mod.rs b/beacon-file-formats/beacon-nd-arrow/src/array/mod.rs index ac1fd160..366432fb 100644 --- a/beacon-file-formats/beacon-nd-arrow/src/array/mod.rs +++ b/beacon-file-formats/beacon-nd-arrow/src/array/mod.rs @@ -115,11 +115,7 @@ impl> NdArrowArrayDispatch { Self::from_parts(Arc::new(backend), shape, dimensions) } - fn from_parts( - backend: Arc, - shape: Vec, - dimensions: Vec, - ) -> Result { + fn from_parts(backend: Arc, shape: Vec, dimensions: Vec) -> Result { if shape.len() != dimensions.len() { return Err(NdArrowError::ShapeDimensionMismatch { shape_len: shape.len(), @@ -259,7 +255,11 @@ impl> NdArrowArray for NdArrowArrayDi let broadcasted = aligned.broadcast(target_shape).ok_or_else(|| { let source_shape = self.shape(); - tracing::warn!(?source_shape, ?target_shape, "cannot broadcast array to target shape"); + tracing::warn!( + ?source_shape, + ?target_shape, + "cannot broadcast array to target shape" + ); anyhow::anyhow!( "Cannot broadcast array of shape {:?} to target shape {:?}", source_shape, diff --git a/beacon-file-formats/beacon-nd-arrow/src/lib.rs b/beacon-file-formats/beacon-nd-arrow/src/lib.rs index 9b6d2bdc..8b354a56 100644 --- a/beacon-file-formats/beacon-nd-arrow/src/lib.rs +++ b/beacon-file-formats/beacon-nd-arrow/src/lib.rs @@ -15,6 +15,6 @@ pub mod error; pub mod stream; pub use array::NdArrowArrayDispatch; -pub use error::NdArrowError; pub use batch::NdRecordBatch; +pub use error::NdArrowError; pub use stream::{NdToArrowPipeOptions, pipe_nd_record_batch_stream}; diff --git a/beacon-functions/src/blue_cloud/cmems/map_bigram_l05.rs b/beacon-functions/src/blue_cloud/cmems/map_bigram_l05.rs index 76bf47db..5d93e548 100644 --- a/beacon-functions/src/blue_cloud/cmems/map_bigram_l05.rs +++ b/beacon-functions/src/blue_cloud/cmems/map_bigram_l05.rs @@ -77,10 +77,7 @@ mod tests { #[test] fn impl_array_path() { - let input = ColumnarValue::Array(Arc::new(StringArray::from(vec![ - Some("CT"), - Some("ZZ"), - ]))); + let input = ColumnarValue::Array(Arc::new(StringArray::from(vec![Some("CT"), Some("ZZ")]))); let ColumnarValue::Array(arr) = map_cmems_bigram_l05_impl(&[input]).unwrap() else { panic!("expected array"); }; diff --git a/beacon-functions/src/blue_cloud/cmems/map_bigram_l06.rs b/beacon-functions/src/blue_cloud/cmems/map_bigram_l06.rs index 1606103d..e545a871 100644 --- a/beacon-functions/src/blue_cloud/cmems/map_bigram_l06.rs +++ b/beacon-functions/src/blue_cloud/cmems/map_bigram_l06.rs @@ -61,8 +61,10 @@ fn map_cmems_bigram_l06_impl( ColumnarValue::Scalar(ScalarValue::Utf8(bigram)), ColumnarValue::Array(wmo_inst_type_arr), ) => { - let wmo_inst_type_array = - crate::util::downcast_arg::(wmo_inst_type_arr, "map_cmems_bigram_l06")?; + let wmo_inst_type_array = crate::util::downcast_arg::( + wmo_inst_type_arr, + "map_cmems_bigram_l06", + )?; Ok(ColumnarValue::Array(Arc::new(StringArray::from_iter( wmo_inst_type_array @@ -127,7 +129,10 @@ mod tests { #[test] fn mapping_depends_on_wmo_type_only_for_ct() { assert_eq!(map_bigram_l06(Some("CT"), Some("995")), Some("SDN:L06::70")); - assert_eq!(map_bigram_l06(Some("CT"), Some("other")), Some("SDN:L06::30")); + assert_eq!( + map_bigram_l06(Some("CT"), Some("other")), + Some("SDN:L06::30") + ); assert_eq!(map_bigram_l06(Some("BO"), None), Some("SDN:L06::30")); assert_eq!(map_bigram_l06(Some("XX"), None), Some("SDN:L06::0")); assert_eq!(map_bigram_l06(Some("ZZ"), None), None); @@ -143,8 +148,10 @@ mod tests { fn impl_array_array_path() { let bigrams = ColumnarValue::Array(Arc::new(StringArray::from(vec![Some("BO"), Some("ZZ")]))); - let wmo = - ColumnarValue::Array(Arc::new(StringArray::from(vec![None::<&str>, None::<&str>]))); + let wmo = ColumnarValue::Array(Arc::new(StringArray::from(vec![ + None::<&str>, + None::<&str>, + ]))); let ColumnarValue::Array(arr) = map_cmems_bigram_l06_impl(&[bigrams, wmo]).unwrap() else { panic!("expected array"); }; diff --git a/beacon-functions/src/blue_cloud/common/map_measuring_area_type_feature_type.rs b/beacon-functions/src/blue_cloud/common/map_measuring_area_type_feature_type.rs index 3a5fb33e..4105dd79 100644 --- a/beacon-functions/src/blue_cloud/common/map_measuring_area_type_feature_type.rs +++ b/beacon-functions/src/blue_cloud/common/map_measuring_area_type_feature_type.rs @@ -66,8 +66,14 @@ mod tests { #[test] fn feature_type_keywords() { - assert_eq!(map_str_feature_type("a curve segment"), Some("trajectory".to_string())); - assert_eq!(map_str_feature_type("a single point"), Some("profile".to_string())); + assert_eq!( + map_str_feature_type("a curve segment"), + Some("trajectory".to_string()) + ); + assert_eq!( + map_str_feature_type("a single point"), + Some("profile".to_string()) + ); assert_eq!(map_str_feature_type("neither keyword"), None); } diff --git a/beacon-functions/src/blue_cloud/common/map_p01_p25.rs b/beacon-functions/src/blue_cloud/common/map_p01_p25.rs index e69de29b..8b137891 100644 --- a/beacon-functions/src/blue_cloud/common/map_p01_p25.rs +++ b/beacon-functions/src/blue_cloud/common/map_p01_p25.rs @@ -0,0 +1 @@ + diff --git a/beacon-functions/src/blue_cloud/common/map_p25_l05.rs b/beacon-functions/src/blue_cloud/common/map_p25_l05.rs index e69de29b..8b137891 100644 --- a/beacon-functions/src/blue_cloud/common/map_p25_l05.rs +++ b/beacon-functions/src/blue_cloud/common/map_p25_l05.rs @@ -0,0 +1 @@ + diff --git a/beacon-functions/src/blue_cloud/common/pressure_to_depth_teos_10.rs b/beacon-functions/src/blue_cloud/common/pressure_to_depth_teos_10.rs index 1a36defd..91b8518d 100644 --- a/beacon-functions/src/blue_cloud/common/pressure_to_depth_teos_10.rs +++ b/beacon-functions/src/blue_cloud/common/pressure_to_depth_teos_10.rs @@ -105,7 +105,10 @@ mod tests { // ~100 dbar is roughly ~99 m of depth; assert it is positive and close. let depth = gsw_depth_from_pressure(100.0, 30.0); assert!(depth > 0.0, "depth should be positive, got {depth}"); - assert!((depth - 99.0).abs() < 5.0, "depth out of expected range: {depth}"); + assert!( + (depth - 99.0).abs() < 5.0, + "depth out of expected range: {depth}" + ); } fn f64_array(out: ColumnarValue) -> Float64Array { diff --git a/beacon-functions/src/blue_cloud/cora/map_platform_l06.rs b/beacon-functions/src/blue_cloud/cora/map_platform_l06.rs index 8e011dff..adfadd65 100644 --- a/beacon-functions/src/blue_cloud/cora/map_platform_l06.rs +++ b/beacon-functions/src/blue_cloud/cora/map_platform_l06.rs @@ -60,8 +60,10 @@ fn map_cora_platform_l06_impl( ColumnarValue::Scalar(ScalarValue::Utf8(bigram)), ColumnarValue::Array(wmo_inst_type_arr), ) => { - let wmo_inst_type_array = - crate::util::downcast_arg::(wmo_inst_type_arr, "map_cora_platform_l06")?; + let wmo_inst_type_array = crate::util::downcast_arg::( + wmo_inst_type_arr, + "map_cora_platform_l06", + )?; Ok(ColumnarValue::Array(Arc::new(StringArray::from_iter( wmo_inst_type_array.iter().map(|wmo_inst_type| { @@ -125,10 +127,22 @@ mod tests { #[test] fn known_bigrams_map_to_l06_codes() { - assert_eq!(map_cora_platform_func_impl(Some("BO"), None), Some("SDN:L06::30")); - assert_eq!(map_cora_platform_func_impl(Some("DB"), None), Some("SDN:L06::42")); - assert_eq!(map_cora_platform_func_impl(Some("GL"), None), Some("SDN:L06::27")); - assert_eq!(map_cora_platform_func_impl(Some("XX"), None), Some("SDN:L06::0")); + assert_eq!( + map_cora_platform_func_impl(Some("BO"), None), + Some("SDN:L06::30") + ); + assert_eq!( + map_cora_platform_func_impl(Some("DB"), None), + Some("SDN:L06::42") + ); + assert_eq!( + map_cora_platform_func_impl(Some("GL"), None), + Some("SDN:L06::27") + ); + assert_eq!( + map_cora_platform_func_impl(Some("XX"), None), + Some("SDN:L06::0") + ); } #[test] @@ -158,10 +172,8 @@ mod tests { #[test] fn udf_handles_array_array() { - let bigrams = ColumnarValue::Array(Arc::new(StringArray::from(vec![ - Some("BO"), - Some("ZZ"), - ]))); + let bigrams = + ColumnarValue::Array(Arc::new(StringArray::from(vec![Some("BO"), Some("ZZ")]))); let wmo = ColumnarValue::Array(Arc::new(StringArray::from(vec![ None::<&str>, None::<&str>, @@ -194,6 +206,9 @@ mod tests { let bigram = ColumnarValue::Scalar(ScalarValue::Utf8(Some("ZZ".into()))); let wmo = ColumnarValue::Scalar(ScalarValue::Utf8(None)); let out = map_cora_platform_l06_impl(&[bigram, wmo]).unwrap(); - assert!(matches!(out, ColumnarValue::Scalar(ScalarValue::Utf8(None)))); + assert!(matches!( + out, + ColumnarValue::Scalar(ScalarValue::Utf8(None)) + )); } } diff --git a/beacon-functions/src/blue_cloud/emodnet_chemistry/map_instrument_l05_multi.rs b/beacon-functions/src/blue_cloud/emodnet_chemistry/map_instrument_l05_multi.rs index d3a1d15f..cb7f6ba5 100644 --- a/beacon-functions/src/blue_cloud/emodnet_chemistry/map_instrument_l05_multi.rs +++ b/beacon-functions/src/blue_cloud/emodnet_chemistry/map_instrument_l05_multi.rs @@ -96,8 +96,14 @@ mod tests { #[test] fn skips_empty_groups_and_stops_at_unclosed_parenthesis() { // Empty () is dropped; an unclosed '(' ends the scan. - assert_eq!(extract_parenthesized_values_ref("a () b (7)"), vec!["SDN:L05::7"]); - assert_eq!(extract_parenthesized_values_ref("a (7) b (oops"), vec!["SDN:L05::7"]); + assert_eq!( + extract_parenthesized_values_ref("a () b (7)"), + vec!["SDN:L05::7"] + ); + assert_eq!( + extract_parenthesized_values_ref("a (7) b (oops"), + vec!["SDN:L05::7"] + ); assert!(extract_parenthesized_values_ref("no groups").is_empty()); } diff --git a/beacon-functions/src/blue_cloud/emodnet_chemistry/map_platform_l06.rs b/beacon-functions/src/blue_cloud/emodnet_chemistry/map_platform_l06.rs index d688619f..830d2328 100644 --- a/beacon-functions/src/blue_cloud/emodnet_chemistry/map_platform_l06.rs +++ b/beacon-functions/src/blue_cloud/emodnet_chemistry/map_platform_l06.rs @@ -90,8 +90,7 @@ mod tests { fn impl_array_path() { let input = ColumnarValue::Array(Arc::new(StringArray::from(vec![Some("Y (30)"), Some("x")]))); - let ColumnarValue::Array(arr) = - map_emodnet_chemistry_platform_l06_impl(&[input]).unwrap() + let ColumnarValue::Array(arr) = map_emodnet_chemistry_platform_l06_impl(&[input]).unwrap() else { panic!("expected array"); }; diff --git a/beacon-functions/src/blue_cloud/seadatanet/map_originator_edmo.rs b/beacon-functions/src/blue_cloud/seadatanet/map_originator_edmo.rs index 446593a9..3375c3aa 100644 --- a/beacon-functions/src/blue_cloud/seadatanet/map_originator_edmo.rs +++ b/beacon-functions/src/blue_cloud/seadatanet/map_originator_edmo.rs @@ -88,7 +88,10 @@ mod tests { #[test] fn scalar_single_parentheses() { - assert_eq!(scalar_result(Some("Some Institute (123)")), Some("123".to_string())); + assert_eq!( + scalar_result(Some("Some Institute (123)")), + Some("123".to_string()) + ); } #[test] diff --git a/beacon-functions/src/blue_cloud/seadatanet/map_platform_l06.rs b/beacon-functions/src/blue_cloud/seadatanet/map_platform_l06.rs index 517171d4..a9d3e42b 100644 --- a/beacon-functions/src/blue_cloud/seadatanet/map_platform_l06.rs +++ b/beacon-functions/src/blue_cloud/seadatanet/map_platform_l06.rs @@ -90,8 +90,7 @@ mod tests { fn impl_array_path() { let input = ColumnarValue::Array(Arc::new(StringArray::from(vec![Some("Q (27)"), Some("x")]))); - let ColumnarValue::Array(arr) = map_seadatanet_platform_l06_impl(&[input]).unwrap() - else { + let ColumnarValue::Array(arr) = map_seadatanet_platform_l06_impl(&[input]).unwrap() else { panic!("expected array"); }; let arr = arr.as_any().downcast_ref::().unwrap(); diff --git a/beacon-functions/src/blue_cloud/seadatanet/map_units.rs b/beacon-functions/src/blue_cloud/seadatanet/map_units.rs index a2aa190b..983d59f9 100644 --- a/beacon-functions/src/blue_cloud/seadatanet/map_units.rs +++ b/beacon-functions/src/blue_cloud/seadatanet/map_units.rs @@ -108,30 +108,23 @@ impl ScalarUDFImpl for MapUnits { let from_unit = match arg0 { ColumnarValue::Array(array) => array, - ColumnarValue::Scalar(scalar_value) => { - scalar_value.to_array_of_size(number_rows)? - } + ColumnarValue::Scalar(scalar_value) => scalar_value.to_array_of_size(number_rows)?, }; let to_unit = match arg1 { ColumnarValue::Array(array) => array, - ColumnarValue::Scalar(scalar_value) => { - scalar_value.to_array_of_size(number_rows)? - } + ColumnarValue::Scalar(scalar_value) => scalar_value.to_array_of_size(number_rows)?, }; let values = match arg2 { ColumnarValue::Array(array) => array, - ColumnarValue::Scalar(scalar_value) => { - scalar_value.to_array_of_size(number_rows)? - } + ColumnarValue::Scalar(scalar_value) => scalar_value.to_array_of_size(number_rows)?, }; let from_unit = crate::util::downcast_arg::(&from_unit, "map_units")?; let to_unit = crate::util::downcast_arg::(&to_unit, "map_units")?; - let values = - crate::util::downcast_arg::(&values, "map_units")?; + let values = crate::util::downcast_arg::(&values, "map_units")?; let array = PrimitiveArray::::from_iter( values diff --git a/beacon-functions/src/blue_cloud/world_ocean_database/map_quality_flag.rs b/beacon-functions/src/blue_cloud/world_ocean_database/map_quality_flag.rs index f61c2a41..b01407c3 100644 --- a/beacon-functions/src/blue_cloud/world_ocean_database/map_quality_flag.rs +++ b/beacon-functions/src/blue_cloud/world_ocean_database/map_quality_flag.rs @@ -42,8 +42,10 @@ fn map_wod_quality_flag_impl( ) -> datafusion::error::Result { match ¶meters[0] { ColumnarValue::Array(flag) => { - let flag_array = - crate::util::downcast_arg::(flag, "map_wod_quality_flag")?; + let flag_array = crate::util::downcast_arg::( + flag, + "map_wod_quality_flag", + )?; let array = flag_array.iter().map(|flag| { flag.map(|wod_flag| WOD_FLAG_TO_SDN.get(&wod_flag).map(|s| s).cloned()) @@ -86,7 +88,8 @@ mod tests { #[test] fn impl_array_path() { - let input = ColumnarValue::Array(Arc::new(Int64Array::from(vec![Some(0), Some(6), Some(99)]))); + let input = + ColumnarValue::Array(Arc::new(Int64Array::from(vec![Some(0), Some(6), Some(99)]))); let ColumnarValue::Array(arr) = map_wod_quality_flag_impl(&[input]).unwrap() else { panic!("expected array"); }; diff --git a/beacon-functions/src/file_formats/mod.rs b/beacon-functions/src/file_formats/mod.rs index 0104f1d9..86a36235 100644 --- a/beacon-functions/src/file_formats/mod.rs +++ b/beacon-functions/src/file_formats/mod.rs @@ -30,11 +30,13 @@ pub fn register_table_functions( session_ctx.clone(), data_object_store_url.clone(), )), - Arc::new(beacon_arrow_geoparquet::datafusion::ReadGeoParquetFunc::new( - runtime_handle.clone(), - session_ctx.clone(), - data_object_store_url.clone(), - )), + Arc::new( + beacon_arrow_geoparquet::datafusion::ReadGeoParquetFunc::new( + runtime_handle.clone(), + session_ctx.clone(), + data_object_store_url.clone(), + ), + ), Arc::new(beacon_arrow_ipc::datafusion::ReadArrowFunc::new( runtime_handle.clone(), session_ctx.clone(), diff --git a/beacon-functions/src/file_formats/read_schema.rs b/beacon-functions/src/file_formats/read_schema.rs index bdac7e9f..5ecc2c74 100644 --- a/beacon-functions/src/file_formats/read_schema.rs +++ b/beacon-functions/src/file_formats/read_schema.rs @@ -1,17 +1,17 @@ use std::sync::Arc; use arrow::datatypes::{DataType, Field}; +use beacon_arrow_bbf::datafusion::BBFFormat; +use beacon_arrow_csv::datafusion::CsvFormat; +use beacon_arrow_ipc::datafusion::ArrowFormat; use beacon_arrow_netcdf::datafusion::NetcdfFormat; +use beacon_arrow_parquet::datafusion::ParquetFormat; use beacon_arrow_tiff::datafusion::TiffFormat; +use beacon_arrow_zarr::datafusion::ZarrFormat; use beacon_common::{ listing_url::parse_listing_table_url, schema_table_provider::SchemaTableProvider, super_table::SuperListingTable, }; -use beacon_arrow_bbf::datafusion::BBFFormat; -use beacon_arrow_csv::datafusion::CsvFormat; -use beacon_arrow_ipc::datafusion::ArrowFormat; -use beacon_arrow_parquet::datafusion::ParquetFormat; -use beacon_arrow_zarr::datafusion::ZarrFormat; use beacon_object_storage::DatasetsStore; use datafusion::{ catalog::TableFunctionImpl, diff --git a/beacon-functions/src/function_doc.rs b/beacon-functions/src/function_doc.rs index 56557305..585b2e68 100644 --- a/beacon-functions/src/function_doc.rs +++ b/beacon-functions/src/function_doc.rs @@ -198,9 +198,11 @@ mod tests { vec![DataType::Utf8], DataType::Boolean, Volatility::Immutable, - Arc::new(|_: &[ColumnarValue]| -> datafusion::error::Result { - unimplemented!() - }), + Arc::new( + |_: &[ColumnarValue]| -> datafusion::error::Result { + unimplemented!() + }, + ), ); let docs = FunctionDoc::from_scalar(&udf); diff --git a/beacon-functions/src/geo/st_within_point.rs b/beacon-functions/src/geo/st_within_point.rs index 21cb6708..62e63075 100644 --- a/beacon-functions/src/geo/st_within_point.rs +++ b/beacon-functions/src/geo/st_within_point.rs @@ -72,52 +72,51 @@ impl ScalarUDFImpl for WithinPointUdf { "st_within_point expects a float64 array as its third argument".to_string(), ))?; - let mut geom_iter: Box>> = match &args.args[0] { - datafusion::logical_expr::ColumnarValue::Array(array) => { - if let Some(array) = array.as_string_opt::() { - Box::new(array.iter()) - } else { - return Err(datafusion::error::DataFusionError::Internal( - "st_within_point expects a string array as its first argument".to_string(), - )); + let mut geom_iter: Box>> = + match &args.args[0] { + datafusion::logical_expr::ColumnarValue::Array(array) => { + if let Some(array) = array.as_string_opt::() { + Box::new(array.iter()) + } else { + return Err(datafusion::error::DataFusionError::Internal( + "st_within_point expects a string array as its first argument" + .to_string(), + )); + } } - } - datafusion::logical_expr::ColumnarValue::Scalar(scalar_value) => { - if let ScalarValue::Utf8(wkt) = scalar_value { - if let Some(wkt) = wkt { - let wkt = - Wkt::from_str(wkt) - .map_err(|e| anyhow::anyhow!(e)) - .map_err(|e| { - datafusion::error::DataFusionError::Execution(e.to_string()) - })?; - let geometry: Geometry = wkt.try_into().map_err(|e| { - datafusion::error::DataFusionError::Execution(format!( - "st_within_point: invalid WKT geometry: {e:?}" - )) - })?; - let result = st_within_point_fast( - geometry, - &mut lon_iter, - &mut lat_iter, - self.cache_size, - ) - .map_err(|e| { - datafusion::error::DataFusionError::Execution(e.to_string()) - })?; - return Ok(ColumnarValue::Array(Arc::new( - arrow::array::BooleanArray::from(result), - ))); + datafusion::logical_expr::ColumnarValue::Scalar(scalar_value) => { + if let ScalarValue::Utf8(wkt) = scalar_value { + if let Some(wkt) = wkt { + let wkt = Wkt::from_str(wkt).map_err(|e| anyhow::anyhow!(e)).map_err( + |e| datafusion::error::DataFusionError::Execution(e.to_string()), + )?; + let geometry: Geometry = wkt.try_into().map_err(|e| { + datafusion::error::DataFusionError::Execution(format!( + "st_within_point: invalid WKT geometry: {e:?}" + )) + })?; + let result = st_within_point_fast( + geometry, + &mut lon_iter, + &mut lat_iter, + self.cache_size, + ) + .map_err(|e| { + datafusion::error::DataFusionError::Execution(e.to_string()) + })?; + return Ok(ColumnarValue::Array(Arc::new( + arrow::array::BooleanArray::from(result), + ))); + } + // Fallback to repeating the WKT string + Box::new(std::iter::repeat_n(wkt.as_deref(), args.number_rows)) + } else { + return Err(datafusion::error::DataFusionError::Internal( + "st_within_point expects a string as its first argument".to_string(), + )); } - // Fallback to repeating the WKT string - Box::new(std::iter::repeat_n(wkt.as_deref(), args.number_rows)) - } else { - return Err(datafusion::error::DataFusionError::Internal( - "st_within_point expects a string as its first argument".to_string(), - )); } - } - }; + }; let result = st_within_point(&mut geom_iter, &mut lon_iter, &mut lat_iter) .map_err(|e| datafusion::error::DataFusionError::Internal(e.to_string()))?; @@ -147,8 +146,9 @@ fn st_within_point_impl( (Some(geom), Some(lon), Some(lat)) => { // ST_WithinPoint implementation let wkt = Wkt::from_str(geom).map_err(|e| anyhow::anyhow!(e))?; - let geometry: Geometry = - wkt.try_into().map_err(|e| anyhow::anyhow!("invalid WKT geometry: {e:?}"))?; + let geometry: Geometry = wkt + .try_into() + .map_err(|e| anyhow::anyhow!("invalid WKT geometry: {e:?}"))?; let point = geo::Point::new(lon, lat); Ok(geometry.contains(&point)) @@ -163,9 +163,8 @@ fn st_within_point_fast( lat: &mut dyn Iterator>, cache_size: usize, ) -> anyhow::Result> { - let mut cache: lru::LruCache>, bool> = lru::LruCache::new( - NonZero::new(cache_size).expect("Cache size must be non-zero"), - ); + let mut cache: lru::LruCache>, bool> = + lru::LruCache::new(NonZero::new(cache_size).expect("Cache size must be non-zero")); let bounding_rect = geom.bounding_rect(); lon.zip(lat) .map(|(lon, lat)| st_within_point_fast_impl(&geom, bounding_rect, &mut cache, lon, lat)) @@ -273,13 +272,8 @@ mod tests { let lons = [Some(5.0), Some(20.0), None]; let lats = [Some(5.0), Some(20.0), Some(5.0)]; - let result = st_within_point_fast( - geom, - &mut lons.into_iter(), - &mut lats.into_iter(), - 16, - ) - .unwrap(); + let result = + st_within_point_fast(geom, &mut lons.into_iter(), &mut lats.into_iter(), 16).unwrap(); assert_eq!(result, vec![true, false, false]); } @@ -327,7 +321,8 @@ mod tests { let geom = geometry(SQUARE); let bounding_rect = geom.bounding_rect(); let mut cache = lru::LruCache::new(NonZero::new(4).unwrap()); - assert!(!st_within_point_fast_impl(&geom, bounding_rect, &mut cache, None, Some(5.0)) - .unwrap()); + assert!( + !st_within_point_fast_impl(&geom, bounding_rect, &mut cache, None, Some(5.0)).unwrap() + ); } } diff --git a/beacon-functions/src/metadata/helpers.rs b/beacon-functions/src/metadata/helpers.rs index 87e20343..a7b7db63 100644 --- a/beacon-functions/src/metadata/helpers.rs +++ b/beacon-functions/src/metadata/helpers.rs @@ -1,5 +1,5 @@ use arrow::datatypes::FieldRef; -use datafusion::common::{ColumnStatistics, stats::Precision}; +use datafusion::common::{stats::Precision, ColumnStatistics}; use datafusion::scalar::ScalarValue; /// One column's worth of statistics as plain strings, ready to push into Arrow arrays. diff --git a/beacon-functions/src/metadata/view_dataset_statistics.rs b/beacon-functions/src/metadata/view_dataset_statistics.rs index 775ddc3d..f9cdceaa 100644 --- a/beacon-functions/src/metadata/view_dataset_statistics.rs +++ b/beacon-functions/src/metadata/view_dataset_statistics.rs @@ -44,8 +44,8 @@ use datafusion::{ }; use object_store::{ObjectMeta, ObjectStore, ObjectStoreExt}; +use super::helpers::{column_stat_rows, ColumnStatRow}; use crate::file_formats::BeaconTableFunctionImpl; -use super::helpers::{ColumnStatRow, column_stat_rows}; fn output_schema() -> SchemaRef { Arc::new(Schema::new(vec![ @@ -186,11 +186,21 @@ fn build_record_batch( RecordBatch::try_new( output_schema(), vec![ - Arc::new(StringArray::from_iter(rows.iter().map(|r| r.column_name.as_deref()))), - Arc::new(StringArray::from_iter(rows.iter().map(|r| r.data_type.as_deref()))), - Arc::new(StringArray::from_iter(rows.iter().map(|r| r.min_value.as_deref()))), - Arc::new(StringArray::from_iter(rows.iter().map(|r| r.max_value.as_deref()))), - Arc::new(BooleanArray::from(rows.iter().map(|r| r.is_exact).collect::>())), + Arc::new(StringArray::from_iter( + rows.iter().map(|r| r.column_name.as_deref()), + )), + Arc::new(StringArray::from_iter( + rows.iter().map(|r| r.data_type.as_deref()), + )), + Arc::new(StringArray::from_iter( + rows.iter().map(|r| r.min_value.as_deref()), + )), + Arc::new(StringArray::from_iter( + rows.iter().map(|r| r.max_value.as_deref()), + )), + Arc::new(BooleanArray::from( + rows.iter().map(|r| r.is_exact).collect::>(), + )), ], ) .map_err(|e| plan_datafusion_err!("Failed to build statistics record batch: {e}")) diff --git a/beacon-functions/src/metadata/view_statistics_cache.rs b/beacon-functions/src/metadata/view_statistics_cache.rs index 2c6f8e24..5e4ad89a 100644 --- a/beacon-functions/src/metadata/view_statistics_cache.rs +++ b/beacon-functions/src/metadata/view_statistics_cache.rs @@ -37,8 +37,8 @@ use datafusion::{ use object_store::{ObjectStore, ObjectStoreExt}; use tokio::runtime::Handle; -use crate::file_formats::BeaconTableFunctionImpl; use super::helpers::column_stat_rows; +use crate::file_formats::BeaconTableFunctionImpl; // ─── Output schema ────────────────────────────────────────────────────────── @@ -133,23 +133,22 @@ impl TableFunctionImpl for ViewStatisticsCacheFunc { // Validate each cached entry against the live object store by calling head(). // call() is sync, so we bridge into async via block_in_place. - let validations: Vec = - tokio::task::block_in_place(|| { - self.runtime_handle.block_on(async { - let mut results = Vec::with_capacity(entries.len()); - for (path, cached_meta, _) in &entries { - let is_valid = match store.head(path).await { - Ok(current) => { - current.size == cached_meta.size - && current.last_modified == cached_meta.last_modified - } - Err(_) => false, - }; - results.push(is_valid); - } - results - }) - }); + let validations: Vec = tokio::task::block_in_place(|| { + self.runtime_handle.block_on(async { + let mut results = Vec::with_capacity(entries.len()); + for (path, cached_meta, _) in &entries { + let is_valid = match store.head(path).await { + Ok(current) => { + current.size == cached_meta.size + && current.last_modified == cached_meta.last_modified + } + Err(_) => false, + }; + results.push(is_valid); + } + results + }) + }); // Collect owned path strings first so we can borrow them below. let path_strings: Vec = entries @@ -212,11 +211,8 @@ impl TableFunctionImpl for ViewStatisticsCacheFunc { Arc::new(BooleanArray::from(is_exact)), ], ) - .map_err(|e| { - plan_datafusion_err!("Failed to build statistics cache record batch: {e}") - })?; + .map_err(|e| plan_datafusion_err!("Failed to build statistics cache record batch: {e}"))?; Ok(Arc::new(MemTable::try_new(schema, vec![vec![batch]])?)) } } - diff --git a/beacon-mcp/src/catalog.rs b/beacon-mcp/src/catalog.rs index 40736521..5a546341 100644 --- a/beacon-mcp/src/catalog.rs +++ b/beacon-mcp/src/catalog.rs @@ -7,8 +7,8 @@ use std::sync::Arc; -use beacon_core::extensions::{McpExtension, PresetExtension, PresetFilter, PresetOp}; use beacon_core::api::{SchemaFieldView, SchemaView}; +use beacon_core::extensions::{McpExtension, PresetExtension, PresetFilter, PresetOp}; use beacon_core::runtime::Runtime; use beacon_core::AuthIdentity; use rmcp::model::{Tool, ToolAnnotations}; @@ -142,12 +142,19 @@ fn export_query_recipe(args: &Map) -> anyhow::Result { .ok_or_else(|| anyhow::anyhow!("missing required 'sql' argument"))? .trim(); // MCP is read-only: only allow SELECT / WITH (CTE) exports. - let head = sql.split_whitespace().next().unwrap_or("").to_ascii_uppercase(); + let head = sql + .split_whitespace() + .next() + .unwrap_or("") + .to_ascii_uppercase(); anyhow::ensure!( matches!(head.as_str(), "SELECT" | "WITH"), "export_query only supports read-only SELECT queries" ); - let format = args.get("format").and_then(Value::as_str).unwrap_or("parquet"); + let format = args + .get("format") + .and_then(Value::as_str) + .unwrap_or("parquet"); anyhow::ensure!( matches!(format, "parquet" | "arrow" | "csv"), "unsupported format '{format}'; expected one of: parquet, arrow, csv" @@ -156,8 +163,14 @@ fn export_query_recipe(args: &Map) -> anyhow::Result { let body = json!({ "sql": sql, "output": { "format": format } }); let body_py = serde_json::to_string(&body)?; let (imports, reader) = match format { - "parquet" => ("import io, requests, pandas as pd", "df = pd.read_parquet(io.BytesIO(resp.content))"), - "csv" => ("import io, requests, pandas as pd", "df = pd.read_csv(io.BytesIO(resp.content))"), + "parquet" => ( + "import io, requests, pandas as pd", + "df = pd.read_parquet(io.BytesIO(resp.content))", + ), + "csv" => ( + "import io, requests, pandas as pd", + "df = pd.read_csv(io.BytesIO(resp.content))", + ), "arrow" => ( "import io, requests, pyarrow.ipc as pa_ipc", "df = pa_ipc.open_file(io.BytesIO(resp.content)).read_all().to_pandas()", @@ -305,7 +318,13 @@ fn default_tool_name(table: &str) -> String { // yields a valid tool name (see `beacon_core::extensions::is_valid_tool_name`). let sanitized: String = table .chars() - .map(|c| if c.is_ascii_alphanumeric() || c == '_' || c == '-' { c } else { '_' }) + .map(|c| { + if c.is_ascii_alphanumeric() || c == '_' || c == '-' { + c + } else { + '_' + } + }) .collect(); let mut name = format!("query_{sanitized}"); name.truncate(64); @@ -470,7 +489,10 @@ fn build_table_sql( ); } } - cols.iter().map(|c| quote_ident(c)).collect::>().join(", ") + cols.iter() + .map(|c| quote_ident(c)) + .collect::>() + .join(", ") } _ => default_select(exposed.as_deref()), }; @@ -502,9 +524,11 @@ fn build_table_sql( fn default_select(exposed: Option<&[&str]>) -> String { match exposed { - Some(cols) if !cols.is_empty() => { - cols.iter().map(|&c| quote_ident(c)).collect::>().join(", ") - } + Some(cols) if !cols.is_empty() => cols + .iter() + .map(|&c| quote_ident(c)) + .collect::>() + .join(", "), _ => "*".to_string(), } } @@ -531,10 +555,17 @@ fn render_filter(filter: &PresetFilter) -> anyhow::Result { .as_array() .filter(|a| !a.is_empty()) .ok_or_else(|| anyhow::anyhow!("'in' requires a non-empty array"))?; - let vals = arr.iter().map(render_scalar).collect::>>()?; + let vals = arr + .iter() + .map(render_scalar) + .collect::>>()?; Ok(format!("{col} IN ({})", vals.join(", "))) } - op => Ok(format!("{col} {} {}", op.as_sql(), render_scalar(&filter.value)?)), + op => Ok(format!( + "{col} {} {}", + op.as_sql(), + render_scalar(&filter.value)? + )), } } @@ -542,7 +573,13 @@ fn render_scalar(value: &Value) -> anyhow::Result { Ok(match value { Value::Number(n) => n.to_string(), Value::String(s) => format!("'{}'", s.replace('\'', "''")), - Value::Bool(b) => if *b { "TRUE".into() } else { "FALSE".into() }, + Value::Bool(b) => { + if *b { + "TRUE".into() + } else { + "FALSE".into() + } + } Value::Null => "NULL".into(), other => anyhow::bail!("unsupported filter value: {other}"), }) @@ -573,13 +610,22 @@ mod tests { args.insert("sql".into(), Value::String("SELECT * FROM obs".into())); args.insert("format".into(), Value::String("parquet".into())); let out = export_query_recipe(&args).unwrap(); - assert!(out.contains("/api/query"), "recipe should reference the query endpoint"); - assert!(out.contains("read_parquet"), "parquet snippet should use read_parquet"); + assert!( + out.contains("/api/query"), + "recipe should reference the query endpoint" + ); + assert!( + out.contains("read_parquet"), + "parquet snippet should use read_parquet" + ); assert!(out.contains("\"format\": \"parquet\"")); // WITH (CTE) is allowed; default format is parquet. let mut cte = Map::new(); - cte.insert("sql".into(), Value::String("WITH x AS (SELECT 1) SELECT * FROM x".into())); + cte.insert( + "sql".into(), + Value::String("WITH x AS (SELECT 1) SELECT * FROM x".into()), + ); assert!(export_query_recipe(&cte).unwrap().contains("read_parquet")); // Non-SELECT is rejected (MCP is read-only). @@ -601,14 +647,21 @@ mod tests { fn resolve_columns_merges_types_and_descriptions() { use beacon_core::extensions::{ColumnDoc, ExposedColumn}; let schema = SchemaView { - fields: vec![field("lat", "Float64"), field("depth", "Float64"), field("x", "Int64")], + fields: vec![ + field("lat", "Float64"), + field("depth", "Float64"), + field("x", "Int64"), + ], metadata: Default::default(), }; // No exposed_columns -> all columns, types included, no descriptions. let all = resolve_columns(&schema, Some(&mcp(None))); assert_eq!(all.len(), 3); - assert_eq!((all[0].name.as_str(), all[0].data_type.as_str()), ("lat", "Float64")); + assert_eq!( + (all[0].name.as_str(), all[0].data_type.as_str()), + ("lat", "Float64") + ); // Exposed subset (in order), merging schema type + entry description. let ext = McpExtension { @@ -628,10 +681,17 @@ mod tests { let cols = resolve_columns(&schema, Some(&ext)); assert_eq!(cols.len(), 2); assert_eq!( - (cols[0].name.as_str(), cols[0].data_type.as_str(), cols[0].description.as_deref()), + ( + cols[0].name.as_str(), + cols[0].data_type.as_str(), + cols[0].description.as_deref() + ), ("depth", "Float64", Some("meters")) ); - assert_eq!((cols[1].name.as_str(), cols[1].description.as_deref()), ("lat", None)); + assert_eq!( + (cols[1].name.as_str(), cols[1].description.as_deref()), + ("lat", None) + ); } fn mcp(cols: Option>) -> McpExtension { @@ -654,8 +714,14 @@ mod tests { assert_eq!(guardrails_text(&mcp(None)), None); let mut ext = mcp(None); let mut g = std::collections::BTreeMap::new(); - g.insert("recommended_row_limit".to_string(), serde_json::json!(10000)); - g.insert("note".to_string(), serde_json::json!("filter by time first")); + g.insert( + "recommended_row_limit".to_string(), + serde_json::json!(10000), + ); + g.insert( + "note".to_string(), + serde_json::json!("filter by time first"), + ); ext.guardrails = Some(g); // Rendered as `key: value` pairs (BTreeMap => deterministic key order). assert_eq!( @@ -676,7 +742,8 @@ mod tests { ); let mut args = Map::new(); args.insert("preset".into(), Value::String("shallow".into())); - let sql = build_table_sql("obs", &mcp(Some(vec!["lat", "depth"])), Some(&p), &args).unwrap(); + let sql = + build_table_sql("obs", &mcp(Some(vec!["lat", "depth"])), Some(&p), &args).unwrap(); assert_eq!( sql, r#"SELECT "lat", "depth" FROM "obs" WHERE "depth" BETWEEN 0 AND 10 LIMIT 100"# diff --git a/beacon-mcp/src/server.rs b/beacon-mcp/src/server.rs index e48831e0..22008c9e 100644 --- a/beacon-mcp/src/server.rs +++ b/beacon-mcp/src/server.rs @@ -57,7 +57,9 @@ impl ServerHandler for BeaconMcpServer { Ok(text) => Ok(CallToolResult::success(vec![Content::text(text)])), // Surface tool failures as an error result (not a protocol error) so // the model can read and react to the message. - Err(error) => Ok(CallToolResult::error(vec![Content::text(error.to_string())])), + Err(error) => Ok(CallToolResult::error(vec![Content::text( + error.to_string(), + )])), } } } diff --git a/beacon-object-storage/src/datasets_store.rs b/beacon-object-storage/src/datasets_store.rs index 2f7d8482..ade09ed2 100644 --- a/beacon-object-storage/src/datasets_store.rs +++ b/beacon-object-storage/src/datasets_store.rs @@ -890,7 +890,12 @@ mod tests { #[test] fn s3_object_url_path_style_includes_bucket() { - let url = s3_object_url("https://example.test", "my-bucket", false, &Path::from("a/b.nc")) + let url = s3_object_url( + "https://example.test", + "my-bucket", + false, + &Path::from("a/b.nc"), + ) .unwrap(); assert_eq!(url, "https://example.test/my-bucket/a/b.nc"); } diff --git a/beacon-sql-databases/src/definition.rs b/beacon-sql-databases/src/definition.rs index 8de5af65..e97af724 100644 --- a/beacon-sql-databases/src/definition.rs +++ b/beacon-sql-databases/src/definition.rs @@ -53,10 +53,7 @@ impl SqlDatabaseTableDefinition { /// Decrypt the stored credential using the deployment master key from the /// session's [`beacon_config::Config`] extension. Errors (fails closed) if a /// secret is present but no `BEACON_SECRETS_KEY` is configured. - fn decrypt_password( - &self, - context: &SessionContext, - ) -> anyhow::Result> { + fn decrypt_password(&self, context: &SessionContext) -> anyhow::Result> { let Some(secret) = &self.secret else { return Ok(None); }; @@ -64,7 +61,9 @@ impl SqlDatabaseTableDefinition { .state() .config() .get_extension::() - .ok_or_else(|| anyhow!("Beacon configuration is unavailable; cannot decrypt credentials"))?; + .ok_or_else(|| { + anyhow!("Beacon configuration is unavailable; cannot decrypt credentials") + })?; let key = config.secrets.master_key().ok_or_else(|| { anyhow!( "BEACON_SECRETS_KEY is not set; cannot decrypt stored credentials for table '{}'", diff --git a/beacon-sql-databases/src/secret.rs b/beacon-sql-databases/src/secret.rs index 8dbd860a..5019c11f 100644 --- a/beacon-sql-databases/src/secret.rs +++ b/beacon-sql-databases/src/secret.rs @@ -15,7 +15,8 @@ use chacha20poly1305::{ }; use secrecy::SecretString; -const B64: base64::engine::general_purpose::GeneralPurpose = base64::engine::general_purpose::STANDARD; +const B64: base64::engine::general_purpose::GeneralPurpose = + base64::engine::general_purpose::STANDARD; /// An encrypted secret as persisted in a table definition. Carries only /// ciphertext and a nonce — never plaintext. `Debug` is redacted so the secret diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 00000000..522023f2 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,17 @@ +# Beacon uses the default Rust formatting style. +# +# This is intentional and matches Google's Rust style guide, which mandates +# "rustfmt with its default settings" rather than a customized profile +# (https://google.github.io/styleguide/rust/). The community default *is* the +# Google style, so the entries below simply pin those defaults explicitly. +# +# Only stable-channel options are used here: CI formats with stable rustfmt +# (toolchain 1.91), and nightly-only knobs (import grouping, comment wrapping, +# etc.) would merely emit "unstable features are only available in nightly" +# warnings without taking effect. + +edition = "2024" +max_width = 100 +tab_spaces = 4 +hard_tabs = false +newline_style = "Unix"