From badc33592bcb6c7682125bda49646f978b64b4fc Mon Sep 17 00:00:00 2001 From: Thomas Zarebczan Date: Wed, 14 Oct 2026 11:05:01 -0400 Subject: [PATCH 01/41] Add source-bound transparent history PIR to native and browser wallets --- Cargo.lock | 223 +- Cargo.toml | 3 +- THIRD_PARTY_LICENSES.txt | 3444 +++++++++++---- crates/z-engine/Cargo.toml | 6 + crates/z-engine/src/lib.rs | 11 + crates/z-engine/src/native/chain_identity.rs | 6 +- crates/z-engine/src/native/mod.rs | 7 +- crates/z-engine/src/native/wallet.rs | 8 +- .../src/native/wallet/source_wallet_tests.rs | 8 + .../src/native/wallet/transparent_pir.rs | 376 ++ .../transparent_pir_integration_tests.rs | 363 ++ .../wallet/transparent_pir_source_tests.rs | 346 ++ crates/z-engine/src/transparent_pir.rs | 335 ++ crates/z-engine/src/web/prove.rs | 6 + crates/z-engine/src/web/public_data.rs | 2 +- crates/z-engine/src/web/store.rs | 40 + crates/z-engine/src/web/transparent_pir.rs | 620 +++ crates/z-transparent-pir-v2/Cargo.toml | 29 + crates/z-transparent-pir-v2/DESIGN.md | 58 + crates/z-transparent-pir-v2/LICENSE-APACHE | 201 + .../z-transparent-pir-v2/LICENSE-FIXTURE-MIT | 21 + crates/z-transparent-pir-v2/LICENSE-MIT | 21 + crates/z-transparent-pir-v2/UPSTREAM.md | 60 + crates/z-transparent-pir-v2/src/adapters.rs | 54 + crates/z-transparent-pir-v2/src/client.rs | 456 ++ crates/z-transparent-pir-v2/src/lib.rs | 18 + .../z-transparent-pir-v2/src/memory_store.rs | 499 +++ crates/z-transparent-pir-v2/src/pass.rs | 741 ++++ .../z-transparent-pir-v2/src/serialization.rs | 89 + .../z-transparent-pir-v2/src/shared_store.rs | 113 + crates/z-transparent-pir-v2/src/store.rs | 555 +++ crates/z-transparent-pir-v2/src/sync.rs | 3901 +++++++++++++++++ crates/z-transparent-pir-v2/src/sync_ahead.rs | 497 +++ crates/z-transparent-pir-v2/src/transport.rs | 87 + crates/z-transparent-pir-v2/src/types.rs | 147 + crates/z-transparent-pir-v2/tests/fixture.rs | 405 ++ crates/z-transparent-pir-v2/tests/recovery.rs | 586 +++ .../tools/verify-upstream.py | 27 + .../upstream-sources.json | 11 + crates/z-transparent-pir-v2/upstream.patch | 1299 ++++++ crates/z-wasm/Cargo.toml | 2 + crates/z-wasm/src/lib.rs | 6 + crates/z-wasm/src/private_transparent.rs | 114 + docs/README.md | 1 + docs/SDK.md | 3 +- docs/SUPPORT.md | 1 + docs/TRANSPARENT-PIR.md | 89 + licenses/WASM-dependencies.json | 296 +- licenses/source-adaptations.json | 20 + licenses/upstream/hex_lit-MITNFA.txt | 7 + licenses/upstream/sources.json | 8 + .../upstream/transparent-wallet-pir-LICENSE | 21 + package.json | 6 +- packages/base/package.json | 64 +- packages/core/package.json | 2 +- packages/core/src/index.ts | 11 + packages/gift-cards/package.json | 2 +- packages/passkey/package.json | 2 +- packages/sdk/package.json | 2 +- packages/sdk/src/create-wallet.ts | 10 +- packages/sdk/src/index.ts | 2 + packages/sdk/src/lwd.ts | 4 + packages/sdk/src/private-transparent.ts | 141 + packages/sdk/src/runtime.ts | 2 +- packages/sdk/src/scan-host.ts | 15 +- packages/sdk/src/scan.worker.ts | 11 + packages/sdk/src/wasm-client-context.ts | 3 +- packages/sdk/src/wasm-client-runtime.ts | 7 +- packages/sdk/src/wasm-client-sync.ts | 30 +- packages/sdk/src/wasm-client.ts | 15 +- packages/sdk/src/wasm-shim.d.ts | 2 + packages/sdk/test/private-transparent.test.ts | 110 + scripts/license-inventory.py | 32 +- scripts/wasm-build-inputs.mjs | 2 +- tools/check-transparent-pir-upstream.mjs | 15 + 75 files changed, 15934 insertions(+), 803 deletions(-) create mode 100644 crates/z-engine/src/native/wallet/transparent_pir.rs create mode 100644 crates/z-engine/src/native/wallet/transparent_pir_integration_tests.rs create mode 100644 crates/z-engine/src/native/wallet/transparent_pir_source_tests.rs create mode 100644 crates/z-engine/src/transparent_pir.rs create mode 100644 crates/z-engine/src/web/transparent_pir.rs create mode 100644 crates/z-transparent-pir-v2/Cargo.toml create mode 100644 crates/z-transparent-pir-v2/DESIGN.md create mode 100644 crates/z-transparent-pir-v2/LICENSE-APACHE create mode 100644 crates/z-transparent-pir-v2/LICENSE-FIXTURE-MIT create mode 100644 crates/z-transparent-pir-v2/LICENSE-MIT create mode 100644 crates/z-transparent-pir-v2/UPSTREAM.md create mode 100644 crates/z-transparent-pir-v2/src/adapters.rs create mode 100644 crates/z-transparent-pir-v2/src/client.rs create mode 100644 crates/z-transparent-pir-v2/src/lib.rs create mode 100644 crates/z-transparent-pir-v2/src/memory_store.rs create mode 100644 crates/z-transparent-pir-v2/src/pass.rs create mode 100644 crates/z-transparent-pir-v2/src/serialization.rs create mode 100644 crates/z-transparent-pir-v2/src/shared_store.rs create mode 100644 crates/z-transparent-pir-v2/src/store.rs create mode 100644 crates/z-transparent-pir-v2/src/sync.rs create mode 100644 crates/z-transparent-pir-v2/src/sync_ahead.rs create mode 100644 crates/z-transparent-pir-v2/src/transport.rs create mode 100644 crates/z-transparent-pir-v2/src/types.rs create mode 100644 crates/z-transparent-pir-v2/tests/fixture.rs create mode 100644 crates/z-transparent-pir-v2/tests/recovery.rs create mode 100644 crates/z-transparent-pir-v2/tools/verify-upstream.py create mode 100644 crates/z-transparent-pir-v2/upstream-sources.json create mode 100644 crates/z-transparent-pir-v2/upstream.patch create mode 100644 crates/z-wasm/src/private_transparent.rs create mode 100644 docs/TRANSPARENT-PIR.md create mode 100644 licenses/upstream/hex_lit-MITNFA.txt create mode 100644 licenses/upstream/transparent-wallet-pir-LICENSE create mode 100644 packages/sdk/src/private-transparent.ts create mode 100644 packages/sdk/test/private-transparent.test.ts create mode 100644 tools/check-transparent-pir-upstream.mjs diff --git a/Cargo.lock b/Cargo.lock index 56b1cbf..f169eb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -562,6 +562,61 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -595,6 +650,15 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32637268377fc7b10a8c6d51de3e7fba1ce5dd371a96e342b34e6078db558e7f" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bindgen" version = "0.71.1" @@ -3971,6 +4035,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -4433,6 +4503,16 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "objc2-metal" version = "0.3.2" @@ -4740,6 +4820,17 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pir-control" +version = "0.1.0" +source = "git+https://github.com/valargroup/wallet-pir?rev=c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2#c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" +dependencies = [ + "hex", + "rand 0.8.8", + "serde", + "sha2 0.10.9", +] + [[package]] name = "pir-native" version = "0.1.0" @@ -4752,6 +4843,16 @@ dependencies = [ "reinspiring", ] +[[package]] +name = "pir-observability" +version = "0.1.0" +source = "git+https://github.com/valargroup/wallet-pir?rev=c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2#c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" +dependencies = [ + "axum", + "http-body", + "serde", +] + [[package]] name = "pkg-config" version = "0.3.34" @@ -6024,6 +6125,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.21" @@ -6601,6 +6713,20 @@ dependencies = [ "windows 0.57.0", ] +[[package]] +name = "sysinfo" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252800745060e7b9ffb7b2badbd8b31cfa4aa2e61af879d0a3bf2a317c20217d" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows 0.61.3", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -6665,7 +6791,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand 2.5.0", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -6854,7 +6980,9 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -7098,6 +7226,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.13.1", + "bytes", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -7116,6 +7260,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -7220,6 +7365,39 @@ dependencies = [ "transparent-filter", ] +[[package]] +name = "transparent-shard-server" +version = "0.1.0-rc.4" +source = "git+https://github.com/valargroup/wallet-pir?rev=c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2#c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" +dependencies = [ + "axum", + "base64", + "clap", + "hex", + "http-body", + "ipir-sp", + "memmap2", + "pir-control", + "pir-observability", + "rayon", + "reinspiring", + "rustix 1.1.4", + "serde", + "serde_json", + "sha2 0.10.9", + "sysinfo 0.36.1", + "thiserror 2.0.20", + "tokio", + "tower", + "tower-http", + "tracing", + "tracing-subscriber", + "transparent-events", + "transparent-filter", + "transparent-native", + "transparent-shard", +] + [[package]] name = "transparent-wallet" version = "0.1.0-rc.4" @@ -8556,7 +8734,7 @@ dependencies = [ [[package]] name = "z-desktop" -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" dependencies = [ "anyhow", "gpui", @@ -8570,11 +8748,12 @@ dependencies = [ [[package]] name = "z-engine" -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" dependencies = [ "anyhow", "argon2", "async-trait", + "axum", "base64", "bip39", "blake2b_simd", @@ -8618,8 +8797,11 @@ dependencies = [ "tracing-subscriber", "transparent-events", "transparent-filter", + "transparent-shard", + "transparent-shard-server", "transparent-wallet", "z-enhance-pir", + "z-transparent-pir-v2", "zakura-address", "zakura-bls12-381", "zakura-client-backend", @@ -8641,7 +8823,7 @@ dependencies = [ [[package]] name = "z-enhance-pir" -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" dependencies = [ "base64", "enhance-pir", @@ -8660,7 +8842,7 @@ dependencies = [ [[package]] name = "z-node-launcher" -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" dependencies = [ "anyhow", "clap", @@ -8668,9 +8850,35 @@ dependencies = [ "serde_json", ] +[[package]] +name = "z-transparent-pir-v2" +version = "0.1.0-alpha.12" +dependencies = [ + "axum", + "base64", + "bincode", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tower", + "transparent-events", + "transparent-filter", + "transparent-native", + "transparent-shard", + "transparent-shard-server", + "transparent-wallet", +] + [[package]] name = "z-wasm" -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" dependencies = [ "console_error_panic_hook", "getrandom 0.4.3", @@ -8683,6 +8891,7 @@ dependencies = [ "wasm-bindgen-rayon", "z-engine", "z-enhance-pir", + "z-transparent-pir-v2", ] [[package]] @@ -9432,7 +9641,7 @@ dependencies = [ "rand 0.8.8", "screencapturekit", "screencapturekit-sys", - "sysinfo", + "sysinfo 0.31.4", "tao-core-video-sys", "windows 0.61.3", "windows-capture", diff --git a/Cargo.toml b/Cargo.toml index 3dc86db..f257e14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "crates/z-engine", "crates/z-enhance-pir", + "crates/z-transparent-pir-v2", "crates/z-wasm", "crates/z-node-launcher", "crates/z-desktop", @@ -11,7 +12,7 @@ members = [ exclude = ["vendor/zakura", "vendor/pir"] [workspace.package] -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" edition = "2021" rust-version = "1.91" license = "Apache-2.0" diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt index ee2d9ff..4082282 100644 --- a/THIRD_PARTY_LICENSES.txt +++ b/THIRD_PARTY_LICENSES.txt @@ -2115,6 +2115,135 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +======================================================================== +base58ck 0.1.101 — CC0-1.0 +https://crates.io/crates/base58ck/0.1.101 +Published Cargo license declaration: CC0-1.0 + +--- bitcoin_hashes-CC0.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/CC0-1.0.txt) --- +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. + + ======================================================================== base64 0.22.1 — MIT OR Apache-2.0 https://crates.io/crates/base64/0.22.1 @@ -2379,6 +2508,34 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +======================================================================== +bincode 1.3.3 — MIT +https://crates.io/crates/bincode/1.3.3 + +--- LICENSE.md (https://docs.rs/crate/bincode/1.3.3/source/LICENSE.md) --- +The MIT License (MIT) + +Copyright (c) 2014 Ty Overby + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + ======================================================================== bip32 0.6.0-pre.1 — Apache-2.0 OR MIT https://crates.io/crates/bip32/0.6.0-pre.1 @@ -2741,8 +2898,8 @@ express Statement of Purpose. ======================================================================== -bitcoin_hashes 0.14.101 — CC0-1.0 -https://crates.io/crates/bitcoin_hashes/0.14.101 +bitcoin 0.32.102 — CC0-1.0 +https://crates.io/crates/bitcoin/0.32.102 Published Cargo license declaration: CC0-1.0 --- bitcoin_hashes-CC0.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/CC0-1.0.txt) --- @@ -2870,188 +3027,575 @@ express Statement of Purpose. ======================================================================== -bitflags 2.13.1 — MIT OR Apache-2.0 -https://crates.io/crates/bitflags/2.13.1 +bitcoin-io 0.1.101 — CC0-1.0 +https://crates.io/crates/bitcoin-io/0.1.101 +Published Cargo license declaration: CC0-1.0 ---- LICENSE-APACHE (https://docs.rs/crate/bitflags/2.13.1/source/LICENSE-APACHE) --- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +--- bitcoin_hashes-CC0.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/CC0-1.0.txt) --- +Creative Commons Legal Code -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +CC0 1.0 Universal -1. Definitions. + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +Statement of Purpose - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." +4. Limitations and Disclaimers. - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. +======================================================================== +bitcoin-units 0.1.101 — CC0-1.0 +https://crates.io/crates/bitcoin-units/0.1.101 +Published Cargo license declaration: CC0-1.0 -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: +--- bitcoin_hashes-CC0.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/CC0-1.0.txt) --- +Creative Commons Legal Code - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and +CC0 1.0 Universal - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and +Statement of Purpose - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. -END OF TERMS AND CONDITIONS +4. Limitations and Disclaimers. -APPENDIX: How to apply the Apache License to your work. + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. + + +======================================================================== +bitcoin_hashes 0.14.101 — CC0-1.0 +https://crates.io/crates/bitcoin_hashes/0.14.101 +Published Cargo license declaration: CC0-1.0 + +--- bitcoin_hashes-CC0.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/CC0-1.0.txt) --- +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. + + +======================================================================== +bitflags 2.13.1 — MIT OR Apache-2.0 +https://crates.io/crates/bitflags/2.13.1 + +--- LICENSE-APACHE (https://docs.rs/crate/bitflags/2.13.1/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" @@ -11005,7 +11549,1646 @@ https://crates.io/crates/env_logger/0.11.11 APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +--- LICENSE-MIT (https://docs.rs/crate/env_logger/0.11.11/source/LICENSE-MIT) --- +Copyright (c) Individual contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +equivalent 1.0.2 — Apache-2.0 OR MIT +https://crates.io/crates/equivalent/1.0.2 + +--- LICENSE-APACHE (https://docs.rs/crate/equivalent/1.0.2/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +--- LICENSE-MIT (https://docs.rs/crate/equivalent/1.0.2/source/LICENSE-MIT) --- +Copyright (c) 2016--2023 + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +======================================================================== +f4jumble 0.1.1 — MIT OR Apache-2.0 +https://crates.io/crates/f4jumble/0.1.1 + +--- LICENSE-APACHE (https://docs.rs/crate/f4jumble/0.1.1/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + + +--- LICENSE-MIT (https://docs.rs/crate/f4jumble/0.1.1/source/LICENSE-MIT) --- +The MIT License (MIT) + +Copyright (c) 2021 Electric Coin Company + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +======================================================================== +fastrand 2.5.0 — Apache-2.0 OR MIT +https://crates.io/crates/fastrand/2.5.0 + +--- LICENSE-APACHE (https://docs.rs/crate/fastrand/2.5.0/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +--- LICENSE-MIT (https://docs.rs/crate/fastrand/2.5.0/source/LICENSE-MIT) --- +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +======================================================================== +ff 0.14.0 — MIT/Apache-2.0 +https://crates.io/crates/ff/0.14.0 + +--- LICENSE-APACHE (https://docs.rs/crate/ff/0.14.0/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + + +--- LICENSE-MIT (https://docs.rs/crate/ff/0.14.0/source/LICENSE-MIT) --- +The MIT License (MIT) + +Copyright (c) 2017 Sean Bowe + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +======================================================================== +find-msvc-tools 0.1.11 — MIT OR Apache-2.0 +https://crates.io/crates/find-msvc-tools/0.1.11 + +--- LICENSE-APACHE (https://docs.rs/crate/find-msvc-tools/0.1.11/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +--- LICENSE-MIT (https://docs.rs/crate/find-msvc-tools/0.1.11/source/LICENSE-MIT) --- +Copyright (c) 2014 Alex Crichton + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +======================================================================== +fixedbitset 0.5.7 — MIT OR Apache-2.0 +https://crates.io/crates/fixedbitset/0.5.7 + +--- LICENSE-APACHE (https://docs.rs/crate/fixedbitset/0.5.7/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +--- LICENSE-MIT (https://docs.rs/crate/fixedbitset/0.5.7/source/LICENSE-MIT) --- +Copyright (c) 2015-2017 + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +======================================================================== +flume 0.11.1 — Apache-2.0/MIT +https://crates.io/crates/flume/0.11.1 + +--- LICENSE-APACHE (https://docs.rs/crate/flume/0.11.1/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -11013,7 +13196,7 @@ https://crates.io/crates/env_logger/0.11.11 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -11027,35 +13210,78 @@ https://crates.io/crates/env_logger/0.11.11 See the License for the specific language governing permissions and limitations under the License. +--- LICENSE-MIT (https://docs.rs/crate/flume/0.11.1/source/LICENSE-MIT) --- +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. ---- LICENSE-MIT (https://docs.rs/crate/env_logger/0.11.11/source/LICENSE-MIT) --- -Copyright (c) Individual contributors +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +======================================================================== +foldhash 0.1.5 — Zlib +https://crates.io/crates/foldhash/0.1.5 -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +--- LICENSE (https://docs.rs/crate/foldhash/0.1.5/source/LICENSE) --- +Copyright (c) 2024 Orson Peters -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +This software is provided 'as-is', without any express or implied warranty. In +no event will the authors be held liable for any damages arising from the use of +this software. +Permission is granted to anyone to use this software for any purpose, including +commercial applications, and to alter it and redistribute it freely, subject to +the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim + that you wrote the original software. If you use this software in a product, + an acknowledgment in the product documentation would be appreciated but is + not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. ======================================================================== -equivalent 1.0.2 — Apache-2.0 OR MIT -https://crates.io/crates/equivalent/1.0.2 +fpe 0.6.1 — MIT/Apache-2.0 +https://crates.io/crates/fpe/0.6.1 ---- LICENSE-APACHE (https://docs.rs/crate/equivalent/1.0.2/source/LICENSE-APACHE) --- +--- COPYRIGHT (https://docs.rs/crate/fpe/0.6.1/source/COPYRIGHT) --- +Copyrights in the "fpe" library are retained by their contributors. No +copyright assignment is required to contribute to the "fpe" library. + +The "fpe" library is licensed under either of + + * Apache License, Version 2.0, (see ./LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) + * MIT license (see ./LICENSE-MIT or http://opensource.org/licenses/MIT) + +at your option. + +Unless you explicitly state otherwise, any contribution intentionally +submitted for inclusion in the work by you, as defined in the Apache-2.0 +license, shall be dual licensed as above, without any additional terms or +conditions. + + +--- LICENSE-APACHE (https://docs.rs/crate/fpe/0.6.1/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -11259,9 +13485,7 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/equivalent/1.0.2/source/LICENSE-MIT) --- -Copyright (c) 2016--2023 - +--- LICENSE-MIT (https://docs.rs/crate/fpe/0.6.1/source/LICENSE-MIT) --- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the @@ -11288,218 +13512,13 @@ DEALINGS IN THE SOFTWARE. ======================================================================== -f4jumble 0.1.1 — MIT OR Apache-2.0 -https://crates.io/crates/f4jumble/0.1.1 - ---- LICENSE-APACHE (https://docs.rs/crate/f4jumble/0.1.1/source/LICENSE-APACHE) --- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - +funty 2.0.0 — MIT +https://crates.io/crates/funty/2.0.0 ---- LICENSE-MIT (https://docs.rs/crate/f4jumble/0.1.1/source/LICENSE-MIT) --- -The MIT License (MIT) +--- LICENSE.txt (https://docs.rs/crate/funty/2.0.0/source/LICENSE.txt) --- +MIT License -Copyright (c) 2021 Electric Coin Company +Copyright (c) 2020 myrrlyn (Alexander Payne) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -11508,23 +13527,23 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. ======================================================================== -fastrand 2.5.0 — Apache-2.0 OR MIT -https://crates.io/crates/fastrand/2.5.0 +futures 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures/0.3.34 ---- LICENSE-APACHE (https://docs.rs/crate/fastrand/2.5.0/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -11713,7 +13732,8 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -11728,7 +13748,10 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/fastrand/2.5.0/source/LICENSE-MIT) --- +--- LICENSE-MIT (https://docs.rs/crate/futures/0.3.34/source/LICENSE-MIT) --- +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the @@ -11755,10 +13778,10 @@ DEALINGS IN THE SOFTWARE. ======================================================================== -ff 0.14.0 — MIT/Apache-2.0 -https://crates.io/crates/ff/0.14.0 +futures-channel 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures-channel/0.3.34 ---- LICENSE-APACHE (https://docs.rs/crate/ff/0.14.0/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures-channel/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -11947,7 +13970,8 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -11962,36 +13986,40 @@ See the License for the specific language governing permissions and limitations under the License. +--- LICENSE-MIT (https://docs.rs/crate/futures-channel/0.3.34/source/LICENSE-MIT) --- +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors ---- LICENSE-MIT (https://docs.rs/crate/ff/0.14.0/source/LICENSE-MIT) --- -The MIT License (MIT) - -Copyright (c) 2017 Sean Bowe - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. ======================================================================== -find-msvc-tools 0.1.11 — MIT OR Apache-2.0 -https://crates.io/crates/find-msvc-tools/0.1.11 +futures-core 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures-core/0.3.34 ---- LICENSE-APACHE (https://docs.rs/crate/find-msvc-tools/0.1.11/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures-core/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -12180,7 +14208,8 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12195,8 +14224,9 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/find-msvc-tools/0.1.11/source/LICENSE-MIT) --- -Copyright (c) 2014 Alex Crichton +--- LICENSE-MIT (https://docs.rs/crate/futures-core/0.3.34/source/LICENSE-MIT) --- +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated @@ -12224,10 +14254,10 @@ DEALINGS IN THE SOFTWARE. ======================================================================== -fixedbitset 0.5.7 — MIT OR Apache-2.0 -https://crates.io/crates/fixedbitset/0.5.7 +futures-executor 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures-executor/0.3.34 ---- LICENSE-APACHE (https://docs.rs/crate/fixedbitset/0.5.7/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures-executor/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -12416,7 +14446,8 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12431,242 +14462,10 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/fixedbitset/0.5.7/source/LICENSE-MIT) --- -Copyright (c) 2015-2017 - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - - -======================================================================== -flume 0.11.1 — Apache-2.0/MIT -https://crates.io/crates/flume/0.11.1 - ---- LICENSE-APACHE (https://docs.rs/crate/flume/0.11.1/source/LICENSE-APACHE) --- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +--- LICENSE-MIT (https://docs.rs/crate/futures-executor/0.3.34/source/LICENSE-MIT) --- +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors ---- LICENSE-MIT (https://docs.rs/crate/flume/0.11.1/source/LICENSE-MIT) --- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the @@ -12691,53 +14490,12 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -======================================================================== -foldhash 0.1.5 — Zlib -https://crates.io/crates/foldhash/0.1.5 - ---- LICENSE (https://docs.rs/crate/foldhash/0.1.5/source/LICENSE) --- -Copyright (c) 2024 Orson Peters - -This software is provided 'as-is', without any express or implied warranty. In -no event will the authors be held liable for any damages arising from the use of -this software. - -Permission is granted to anyone to use this software for any purpose, including -commercial applications, and to alter it and redistribute it freely, subject to -the following restrictions: - -1. The origin of this software must not be misrepresented; you must not claim - that you wrote the original software. If you use this software in a product, - an acknowledgment in the product documentation would be appreciated but is - not required. - -2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - -3. This notice may not be removed or altered from any source distribution. ======================================================================== -fpe 0.6.1 — MIT/Apache-2.0 -https://crates.io/crates/fpe/0.6.1 - ---- COPYRIGHT (https://docs.rs/crate/fpe/0.6.1/source/COPYRIGHT) --- -Copyrights in the "fpe" library are retained by their contributors. No -copyright assignment is required to contribute to the "fpe" library. - -The "fpe" library is licensed under either of - - * Apache License, Version 2.0, (see ./LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) - * MIT license (see ./LICENSE-MIT or http://opensource.org/licenses/MIT) +futures-io 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures-io/0.3.34 -at your option. - -Unless you explicitly state otherwise, any contribution intentionally -submitted for inclusion in the work by you, as defined in the Apache-2.0 -license, shall be dual licensed as above, without any additional terms or -conditions. - - ---- LICENSE-APACHE (https://docs.rs/crate/fpe/0.6.1/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures-io/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -12926,7 +14684,8 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12941,7 +14700,10 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/fpe/0.6.1/source/LICENSE-MIT) --- +--- LICENSE-MIT (https://docs.rs/crate/futures-io/0.3.34/source/LICENSE-MIT) --- +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the @@ -12968,38 +14730,10 @@ DEALINGS IN THE SOFTWARE. ======================================================================== -funty 2.0.0 — MIT -https://crates.io/crates/funty/2.0.0 +futures-macro 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures-macro/0.3.34 ---- LICENSE.txt (https://docs.rs/crate/funty/2.0.0/source/LICENSE.txt) --- -MIT License - -Copyright (c) 2020 myrrlyn (Alexander Payne) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -======================================================================== -futures-core 0.3.34 — MIT OR Apache-2.0 -https://crates.io/crates/futures-core/0.3.34 - ---- LICENSE-APACHE (https://docs.rs/crate/futures-core/0.3.34/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures-macro/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -13204,7 +14938,7 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/futures-core/0.3.34/source/LICENSE-MIT) --- +--- LICENSE-MIT (https://docs.rs/crate/futures-macro/0.3.34/source/LICENSE-MIT) --- Copyright (c) 2016 Alex Crichton Copyright (c) 2017 The Tokio Authors @@ -16033,6 +17767,21 @@ express Statement of Purpose. +======================================================================== +hex_lit 0.1.1 — MITNFA +https://crates.io/crates/hex_lit/0.1.1 +Published Cargo license declaration: MITNFA + +--- hex_lit-MITNFA.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/MITNFA.txt) --- +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +Distributions of all or part of the Software intended to be used by the recipients as they would use the unmodified Software, containing modifications that substantially alter, remove, or disable functionality of the Software, outside of the documented configuration mechanisms provided by the Software, shall be modified such that the Original Author's bug reporting email addresses and urls are either replaced with the contact information of the parties responsible for the changes, or removed entirely. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + ======================================================================== hmac 0.13.0-pre.4 — MIT OR Apache-2.0 https://crates.io/crates/hmac/0.13.0-pre.4 @@ -23202,6 +24951,35 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +======================================================================== +pir-native 0.1.0 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/shared/pir-native +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + ======================================================================== poly1305 0.8.0 — Apache-2.0 OR MIT https://crates.io/crates/poly1305/0.8.0 @@ -35764,6 +37542,151 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +======================================================================== +transparent-events 0.1.0-rc.4 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/transparent/crates/transparent-events +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +transparent-filter 0.1.0-rc.4 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/transparent/crates/transparent-filter +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +transparent-native 0.1.0-rc.4 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/transparent/crates/transparent-native +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +transparent-shard 0.1.0-rc.4 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/transparent/crates/transparent-shard +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +transparent-wallet 0.1.0-rc.4 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/transparent/crates/transparent-wallet +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + ======================================================================== typenum 1.20.1 — MIT OR Apache-2.0 https://crates.io/crates/typenum/1.20.1 @@ -46436,3 +48359,30 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +======================================================================== +Adapted transparent PIR asynchronous recovery walker — MIT +Copyright (c) 2025-2026 Roman Akhtariev +Original source revision: c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2 +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/crates/z-engine/Cargo.toml b/crates/z-engine/Cargo.toml index ccb49a8..010fcc3 100644 --- a/crates/z-engine/Cargo.toml +++ b/crates/z-engine/Cargo.toml @@ -59,6 +59,7 @@ lwd = [ sync = ["lwd", "zcash_client_backend/sync"] transparent-inputs = ["zcash_client_backend/transparent-inputs", "dep:transparent"] private-enhance = ["dep:z-enhance-pir"] +transparent-pir = ["transparent-inputs", "dep:z-transparent-pir-v2", "dep:transparent-events", "dep:sha2"] native-pir = ["native", "dep:transparent-wallet", "dep:transparent-events", "dep:transparent-filter", "dep:sha2", "dep:tempfile"] # WASM proving (Orchard/Ironwood in-process). Sapling params stay out of the blob. wasm-prove = ["orchard/circuit", "zcash_primitives/circuits"] @@ -96,6 +97,7 @@ cli = [ ] [dependencies] +z-transparent-pir-v2 = { path = "../z-transparent-pir-v2", optional = true } z-enhance-pir = { path = "../z-enhance-pir", optional = true } anyhow.workspace = true thiserror.workspace = true @@ -180,6 +182,10 @@ getrandom = { version = "0.2", features = ["js"] } time = { version = "0.3.22", default-features = false, features = ["std", "wasm-bindgen"] } [dev-dependencies] +axum = "0.7" +transparent-filter = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-shard-server = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-shard = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } rand_08 = { package = "rand", version = "0.8" } hex = "0.4" enhance-pir = { path = "../../vendor/pir/enhance-pir", default-features = false } diff --git a/crates/z-engine/src/lib.rs b/crates/z-engine/src/lib.rs index 53a8652..459d5c2 100644 --- a/crates/z-engine/src/lib.rs +++ b/crates/z-engine/src/lib.rs @@ -22,6 +22,8 @@ pub mod keys; pub mod ledger; pub(crate) mod offload; pub mod scan; +#[cfg(feature = "transparent-pir")] +pub mod transparent_pir; pub mod web; mod zip321; @@ -56,6 +58,15 @@ pub enum Network { } impl Network { + /// Canonical chain identity, independent of configured endpoints. + pub fn genesis_hash(self) -> &'static str { + match self { + Self::Mainnet => "00040fe8ec8471911baa1db1266ea15dd06b4a8a5c453883c000b031973dce08", + Self::Testnet => "05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38", + Self::Regtest => "029f11d80ef9765602235e1bc9727e3eb6ba20839319f761fee920d63401e327", + } + } + pub fn as_str(self) -> &'static str { match self { Self::Mainnet => "mainnet", diff --git a/crates/z-engine/src/native/chain_identity.rs b/crates/z-engine/src/native/chain_identity.rs index e09bfb6..95e53cf 100644 --- a/crates/z-engine/src/native/chain_identity.rs +++ b/crates/z-engine/src/native/chain_identity.rs @@ -6,11 +6,7 @@ use crate::{ // Zakura chain parameters: checkpoint/list.rs and network/testnet.rs. pub(super) fn genesis(network: Network) -> &'static str { - match network { - Network::Mainnet => "00040fe8ec8471911baa1db1266ea15dd06b4a8a5c453883c000b031973dce08", - Network::Testnet => "05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38", - Network::Regtest => "029f11d80ef9765602235e1bc9727e3eb6ba20839319f761fee920d63401e327", - } + network.genesis_hash() } pub(super) fn validate(network: Network, chain: &str, hash: &str) -> Result<()> { diff --git a/crates/z-engine/src/native/mod.rs b/crates/z-engine/src/native/mod.rs index 28f00e2..7bc0907 100644 --- a/crates/z-engine/src/native/mod.rs +++ b/crates/z-engine/src/native/mod.rs @@ -108,7 +108,7 @@ pub use transparent_wallet::{ #[cfg(feature = "native-pir")] pub use wallet::{ verify_regtest_recovery_blocks, PirConfirmedTransaction, PirDiscoveryReport, PirRecoveryReport, - RecoveryCancellation, RegtestAcceptedChain, VerifiedRegtestRecoveryBlocks, + RegtestAcceptedChain, VerifiedRegtestRecoveryBlocks, }; #[cfg(feature = "native-pir")] @@ -153,3 +153,8 @@ mod replacement_tests { #[cfg(feature = "private-enhance")] pub use wallet::NativeEnhancePass; + +#[cfg(feature = "transparent-pir")] +pub use wallet::NativeTransparentPirPass; +#[cfg(any(feature = "native-pir", feature = "transparent-pir"))] +pub use wallet::RecoveryCancellation; diff --git a/crates/z-engine/src/native/wallet.rs b/crates/z-engine/src/native/wallet.rs index 9967ccb..cd5cd95 100644 --- a/crates/z-engine/src/native/wallet.rs +++ b/crates/z-engine/src/native/wallet.rs @@ -5,19 +5,23 @@ mod database_lease; mod recovery_blocks; #[cfg(feature = "native-pir")] mod recovery_bootstrap; -#[cfg(feature = "native-pir")] +#[cfg(any(feature = "native-pir", feature = "transparent-pir"))] mod recovery_cancellation; #[cfg(feature = "native-pir")] mod recovery_effects; mod recovery_guard; #[cfg(feature = "native-pir")] pub use recovery_blocks::{verify_regtest_recovery_blocks, VerifiedRegtestRecoveryBlocks}; -#[cfg(feature = "native-pir")] +#[cfg(any(feature = "native-pir", feature = "transparent-pir"))] pub use recovery_cancellation::RecoveryCancellation; mod memo_status; mod payments; mod shared_memos; +#[cfg(feature = "transparent-pir")] +mod transparent_pir; pub use shared_memos::NativeSharedMemosPass; +#[cfg(feature = "transparent-pir")] +pub use transparent_pir::NativeTransparentPirPass; #[cfg(feature = "private-enhance")] mod private_enhance; #[cfg(feature = "private-enhance")] diff --git a/crates/z-engine/src/native/wallet/source_wallet_tests.rs b/crates/z-engine/src/native/wallet/source_wallet_tests.rs index 67611f1..af3e7ac 100644 --- a/crates/z-engine/src/native/wallet/source_wallet_tests.rs +++ b/crates/z-engine/src/native/wallet/source_wallet_tests.rs @@ -604,3 +604,11 @@ async fn source_rejection_keeps_saved_intent_and_exposes_only_safe_error_text() assert_eq!(wallet.payment_receipt(&id).unwrap().phase, "unknown"); assert_eq!(*fixture.sent.lock().unwrap(), vec![raw]); } + +#[cfg(feature = "transparent-pir")] +#[path = "transparent_pir_source_tests.rs"] +mod transparent_pir_projection_tests; + +#[cfg(feature = "transparent-pir")] +#[path = "transparent_pir_integration_tests.rs"] +mod transparent_pir_integration_tests; diff --git a/crates/z-engine/src/native/wallet/transparent_pir.rs b/crates/z-engine/src/native/wallet/transparent_pir.rs new file mode 100644 index 0000000..f4ec9aa --- /dev/null +++ b/crates/z-engine/src/native/wallet/transparent_pir.rs @@ -0,0 +1,376 @@ +//! Source-bound canonical transparent application. No address queries or raw-transaction fabrication. +use super::*; +use crate::transparent_pir::{ + parse, refused, ProjectionState, TransparentPirAnchor, TransparentPirCheckpoint, + TransparentPirScope, TransparentPirScript, +}; +use rusqlite::OptionalExtension; +use transparent::{ + address::Script, + bundle::{OutPoint, TxOut}, +}; +use transparent_events::TransparentEvent; +use zcash_client_backend::{ + data_api::ll::{LowLevelWalletRead, LowLevelWalletWrite}, + wallet::{WalletTransparentOutput, WalletTx}, +}; + +pub struct NativeTransparentPirPass { + root: PathBuf, + binding: [u8; 32], + scope: TransparentPirScope, + cancel: Arc, + _owner: std::fs::File, +} +impl NativeTransparentPirPass { + pub fn scope_json(&self) -> Result { + serde_json::to_string(&self.scope).map_err(|_| refused("transparent_pir_scope_invalid")) + } + pub fn cancel(&self) { + self.cancel.cancel(); + } +} +fn database(_: impl std::fmt::Display) -> EngineError { + refused("transparent_pir_database_error") +} +fn load(ext: &rusqlite::Transaction<'_>) -> anyhow::Result { + ext.execute("CREATE TABLE IF NOT EXISTS ext_transparent_pir_v1 (id INTEGER PRIMARY KEY CHECK(id=1), state BLOB NOT NULL)", [])?; + let bytes: Option> = ext + .query_row( + "SELECT state FROM ext_transparent_pir_v1 WHERE id=1", + [], + |r| r.get(0), + ) + .optional()?; + Ok(match bytes { + Some(bytes) => serde_json::from_slice(&bytes)?, + None => ProjectionState::default(), + }) +} +fn save(ext: &rusqlite::Transaction<'_>, state: &ProjectionState) -> anyhow::Result<()> { + ext.execute("INSERT INTO ext_transparent_pir_v1 VALUES(1,?1) ON CONFLICT(id) DO UPDATE SET state=excluded.state", [serde_json::to_vec(state)?])?; + Ok(()) +} +fn scripts>( + db: &D, + tx: &rusqlite::Transaction<'_>, + network: ZNetwork, +) -> Result> +where + D::Error: std::fmt::Display, +{ + let mut result = Vec::new(); + for account in db.get_account_ids().map_err(database)? { + let mut receivers = db + .get_transparent_receivers(account, true, true) + .map_err(database)?; + if db.chain_height().map_err(database)?.is_some() { + receivers.extend( + db.get_ephemeral_transparent_receivers(account, u32::MAX, false) + .map_err(database)?, + ); + } + for (address, _metadata) in receivers { + let script: Script = address.script().into(); + result.push(TransparentPirScript { + script: crate::web::to_hex(&script.0 .0), + required_from: 0, + }); + } + } + // Pinned WalletRead exposes only exposed ephemeral receivers. Include the + // engine's already allocated ephemeral gap without deriving outside Rust. + let mut stmt=tx.prepare("SELECT cached_transparent_receiver_address FROM addresses WHERE key_scope=2 AND cached_transparent_receiver_address IS NOT NULL ORDER BY id LIMIT 4097").map_err(database)?; + for address in stmt + .query_map([], |r| r.get::<_, String>(0)) + .map_err(database)? + { + let address = address.map_err(database)?; + let address = zcash_keys::address::Address::decode(&network, &address) + .and_then(|a| a.to_transparent_address()) + .ok_or_else(|| refused("transparent_pir_scope_invalid"))?; + let script: Script = address.script().into(); + result.push(TransparentPirScript { + script: crate::web::to_hex(&script.0 .0), + required_from: 0, + }); + if result.len() > 8192 { + return Err(refused("transparent_pir_scope_limit")); + } + } + Ok(result) +} +fn retire_projection( + tx: &rusqlite::Transaction<'_>, + state: &mut ProjectionState, + old: &std::collections::BTreeMap, + incoming: &[(Vec, TransparentEvent)], + height: u32, +) -> anyhow::Result<()> { + let incoming: std::collections::BTreeMap<_, _> = incoming + .iter() + .map(|(_, e)| { + ( + crate::transparent_pir::event_key(*e), + crate::transparent_pir::digest(&e.to_bytes()), + ) + }) + .collect(); + let retired: Vec<_> = old + .iter() + .filter(|(key, (h, d))| *h > height && incoming.get(*key) != Some(d)) + .map(|(k, _)| k.clone()) + .collect(); + // Spends retire before outputs. Only relations this source originally created + // are eligible; existing local pending/independent relations are never owned. + for key in &retired { + let fields: Vec<_> = key.split(':').collect(); + if fields.first() == Some(&"s") && state.projected_spends.remove(key) { + let spending = crate::web::from_hex(fields[1]) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + let previous = crate::web::from_hex(fields[3]) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + let index: u32 = fields[4].parse()?; + let local: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM transactions WHERE txid=?1 AND created IS NOT NULL)", + [&spending], + |r| r.get(0), + )?; + if local { + anyhow::bail!("transparent_pir_pending_reconciliation_required"); + } + tx.execute("DELETE FROM transparent_received_output_spends WHERE transaction_id IN(SELECT id_tx FROM transactions WHERE txid=?1) AND transparent_received_output_id IN(SELECT n.id FROM transparent_received_outputs n JOIN transactions p ON p.id_tx=n.transaction_id WHERE p.txid=?2 AND n.output_index=?3)",rusqlite::params![spending,previous,index])?; + tx.execute("DELETE FROM transparent_spend_map WHERE spending_transaction_id IN(SELECT id_tx FROM transactions WHERE txid=?1) AND prevout_txid=?2 AND prevout_output_index=?3",rusqlite::params![spending,previous,index])?; + } + } + for key in &retired { + let fields: Vec<_> = key.split(':').collect(); + if fields.first() == Some(&"r") && state.projected_outputs.remove(key) { + let id = crate::web::from_hex(fields[1]) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + let index: u32 = fields[2].parse()?; + let locked:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid=?1 AND n.output_index=?2 AND (n.lock_owner IS NOT NULL OR n.lock_expiry_height IS NOT NULL OR EXISTS(SELECT 1 FROM transparent_received_output_spends s JOIN transactions p ON p.id_tx=s.transaction_id WHERE s.transparent_received_output_id=n.id AND p.created IS NOT NULL)))",rusqlite::params![id,index],|r|r.get(0))?; + if locked { + anyhow::bail!("transparent_pir_pending_reconciliation_required"); + } + // Retained full bytes are independent wallet evidence; leave those rows. + tx.execute("DELETE FROM transparent_received_outputs WHERE transaction_id IN(SELECT id_tx FROM transactions WHERE txid=?1 AND raw IS NULL AND created IS NULL) AND output_index=?2",rusqlite::params![id,index])?; + } + } + let ids: Vec<_> = state.projected_transactions.iter().cloned().collect(); + for id in ids { + let bytes = crate::web::from_hex(&id) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + // Raw/locally-created transactions retain their independent pending authority. + tx.execute("UPDATE transactions SET mined_height=NULL,block=NULL,tx_index=NULL WHERE txid=?1 AND mined_height>?2 AND raw IS NULL AND created IS NULL",rusqlite::params![bytes,height])?; + } + Ok(()) +} + +struct TransactionConnection<'a, 'b>(&'a rusqlite::Transaction<'b>); +impl std::borrow::Borrow for TransactionConnection<'_, '_> { + fn borrow(&self) -> &rusqlite::Connection { + self.0 + } +} +impl<'a, 'b> std::borrow::Borrow> for TransactionConnection<'a, 'b> { + fn borrow(&self) -> &rusqlite::Transaction<'b> { + self.0 + } +} +type ProjectionDb<'a, 'b> = + WalletDb, ZNetwork, SystemClock, UnwrapErr>; + +impl NativeWallet { + fn transparent_transaction( + &self, + operation: impl FnOnce( + &mut ProjectionDb<'_, '_>, + &rusqlite::Transaction<'_>, + ) -> anyhow::Result, + ) -> Result { + // This engine-owned connection wraps one canonical wallet transaction. + // It is never held over transport and never exposed to the host. + let mut conn = rusqlite::Connection::open(&self.paths.data_db).map_err(database)?; + rusqlite::vtab::array::load_module(&conn).map_err(database)?; + conn.busy_timeout(Duration::from_secs(5)) + .map_err(database)?; + conn.pragma_update(None, "foreign_keys", "ON") + .map_err(database)?; + conn.pragma_update(None, "synchronous", "FULL") + .map_err(database)?; + let tx = conn + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(database)?; + let mut wdb = WalletDb::from_connection( + TransactionConnection(&tx), + self.network, + SystemClock, + new_rng(), + ); + let result = operation(&mut wdb, &tx).map_err(EngineError::from)?; + drop(wdb); + tx.commit().map_err(database)?; + Ok(result) + } + + pub fn begin_transparent_pir_with_source( + &self, + source: &super::super::NativeWalletSource, + source_id: &str, + cancel: Arc, + ) -> Result { + self.require_wallet_source(source)?; + let owner = database_lease::shared(&self.paths.data_db)?; + let scope = cancel.transaction(|| { + self.transparent_transaction(|wdb, ext| -> anyhow::Result<_> { + let block = wdb + .block_fully_scanned()? + .ok_or_else(|| anyhow::anyhow!("transparent_pir_anchor_unavailable"))?; + let anchor = TransparentPirAnchor { + height: u64::from(u32::from(block.block_height())), + hash: block.block_hash().to_string(), + }; + let mut state = load(ext)?; + if let Some(old) = &state.scope { + let known = wdb + .get_block_hash(u32::try_from(old.anchor.height)?.into())? + .map(|h| h.to_string()); + if known.as_deref() != Some(&old.anchor.hash) { + state.rewind(0); + } + } + let mut scope = state.begin(source_id, anchor, scripts(wdb, ext, self.network)?)?; + scope.accepted_headers.retain(|h| h.height != 0); + scope.accepted_headers.push(TransparentPirAnchor { + height: 0, + hash: self.network.genesis_hash().into(), + }); + state.scope = Some(scope.clone()); + save(ext, &state)?; + Ok(scope) + }) + })?; + Ok(NativeTransparentPirPass { + root: self.paths.root.clone(), + binding: source.binding(), + scope, + cancel, + _owner: owner, + }) + } + + pub fn apply_transparent_pir_checkpoint_with_source( + &self, + source: &super::super::NativeWalletSource, + pass: &mut NativeTransparentPirPass, + json: &str, + ) -> Result { + self.require_wallet_source(source)?; + if pass.root != self.paths.root || pass.binding != source.binding() { + return Err(refused("transparent_pir_context_changed")); + } + let checkpoint: TransparentPirCheckpoint = parse(json)?; + if checkpoint.scope_id != pass.scope.scope_id + || checkpoint.generation != pass.scope.generation + { + return Err(refused("transparent_pir_context_changed")); + } + pass.cancel.transaction(|| self.transparent_transaction(|wdb,ext| -> anyhow::Result<_> { + let mut state = load(ext)?; + let old_records = state.records.clone(); + let retirement_floor = state.rewind_floor.into_iter().chain(checkpoint.rewind_to.as_ref().map(|a|a.height as u32)).min(); + if let Some(floor)=retirement_floor { state.prepare_rewind(floor); } + if let Some(ancestor)=&checkpoint.rewind_to { + let height=u32::try_from(ancestor.height)?; + let known=if height==0 {Some(self.network.genesis_hash().to_string())} else {wdb.get_block_hash(height.into())?.map(|h|h.to_string())}; + if known.as_deref()!=Some(&ancestor.hash) {anyhow::bail!("transparent_pir_rewind_anchor_unknown");} + state.prepare_rewind(height); + } + let records = state.validate(&checkpoint)?; + let height = u32::try_from(checkpoint.anchor.height)?; + let block = wdb.block_fully_scanned()?.ok_or_else(||anyhow::anyhow!("transparent_pir_anchor_unavailable"))?; + if u32::from(block.block_height()) != height || block.block_hash().to_string() != checkpoint.anchor.hash { anyhow::bail!("transparent_pir_chain_changed"); } + if state.checkpoint_id.as_ref() == Some(&checkpoint.checkpoint_id) { + state.durable = true; + return Ok(state.status()); + } + let current_scripts = scripts(wdb,ext,self.network)?; + let current_id = crate::transparent_pir::digest(&serde_json::to_vec(&{let mut s=current_scripts.clone();s.sort_by(|a,b|a.script.cmp(&b.script));s.dedup();s})?); + if pass.scope.scripts.iter().any(|enrolled| !current_scripts.contains(enrolled)) { anyhow::bail!("transparent_pir_scope_changed"); } + if retirement_floor.is_some() && !checkpoint.complete { + state.records = old_records; + state.stage(&checkpoint,&[])?; + state.rewind_floor = retirement_floor; + state.durable = true; + save(ext,&state)?; + return Ok(state.status()); + } + if let Some(floor)=retirement_floor { retire_projection(ext,&mut state,&old_records,&records,floor)?; } + for (script,event) in &records { + let (id,position,mined) = match event { TransparentEvent::Receive(e) => (e.txid,e.transaction_index,e.height), TransparentEvent::Spend(e)=>(e.spending_txid,e.transaction_index,e.height) }; + let old:Option<(Option,Option)> = ext.query_row("SELECT mined_height,tx_index FROM transactions WHERE txid=?1",[id.0.as_slice()],|r|Ok((r.get(0)?,r.get(1)?))).optional()?; + if old.is_some_and(|(h,i)|h.is_some_and(|h|h!=mined) || i.is_some_and(|i|i!=u32::from(position))) { anyhow::bail!("transparent_pir_transaction_conflict"); } + if old.is_none() { state.projected_transactions.insert(crate::web::to_hex(&id.0)); } + let meta = WalletTx::new(TxId::from_bytes(id.0),position.into(),vec![],vec![],vec![],vec![],vec![],vec![],vec![]); + let tx_ref = wdb.put_tx_meta(&meta,mined.into())?; + match event { + TransparentEvent::Receive(e) => { + let output = WalletTransparentOutput::from_parts(OutPoint::new(e.txid.0,e.output_index),TxOut::new(Zatoshis::from_u64(e.value)?, Script(zcash_script::script::Code(script.clone()))),Some(e.height.into()),None,None,None).ok_or_else(||anyhow::anyhow!("transparent_pir_script_invalid"))?; + if wdb.find_account_for_transparent_address(output.recipient_address())?.is_none() { anyhow::bail!("transparent_pir_script_not_owned"); } + let old:Option<(Vec,i64,Option)> = ext.query_row("SELECT n.script,n.value_zat,t.mined_height FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid=?1 AND n.output_index=?2",rusqlite::params![e.txid.0.as_slice(),e.output_index],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?; + if old.is_none() {state.projected_outputs.insert(crate::transparent_pir::event_key(*event));} + if old.is_some_and(|(s,v,h)| s != *script || u64::try_from(v).ok()!=Some(e.value) || h.is_some_and(|h|h!=e.height)) { anyhow::bail!("transparent_pir_output_conflict"); } + // Native upstream reconciliation retains existing spend/lock metadata. + wdb.put_transparent_output(&output,height.into(),true)?; + } + TransparentEvent::Spend(e) => { + let prior:bool=ext.query_row("SELECT EXISTS(SELECT 1 FROM transparent_spend_map s JOIN transactions t ON t.id_tx=s.spending_transaction_id WHERE t.txid=?1 AND s.prevout_txid=?2 AND s.prevout_output_index=?3) OR EXISTS(SELECT 1 FROM transparent_received_output_spends s JOIN transactions t ON t.id_tx=s.transaction_id JOIN transparent_received_outputs n ON n.id=s.transparent_received_output_id JOIN transactions p ON p.id_tx=n.transaction_id WHERE t.txid=?1 AND p.txid=?2 AND n.output_index=?3)",rusqlite::params![e.spending_txid.0.as_slice(),e.spent_txid.0.as_slice(),e.spent_output_index],|r|r.get(0))?; + if !prior {state.projected_spends.insert(crate::transparent_pir::event_key(*event));} + let found = wdb.mark_transparent_utxo_spent(&OutPoint::new(e.spent_txid.0,e.spent_output_index),tx_ref)?; + if !found && checkpoint.complete { anyhow::bail!("transparent_pir_unresolved_spend"); } + } + } + } + let unspent:u64=ext.query_row("SELECT COALESCE(SUM(n.value_zat),0) FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.mined_height IS NOT NULL AND NOT EXISTS(SELECT 1 FROM transparent_received_output_spends s WHERE s.transparent_received_output_id=n.id)",[],|r|r.get(0))?; + Zatoshis::from_u64(unspent).map_err(|_|anyhow::anyhow!("transparent_pir_amount_invalid"))?; + state.stage(&checkpoint,&records)?; + let next_scripts = scripts(wdb,ext,self.network)?; + if next_scripts.len() != current_scripts.len() || current_id != pass.scope.scope_id { state.complete=false; } + state.durable=true; + save(ext,&state)?; + Ok(state.status()) + })) + } + + pub fn transparent_pir_status(&self) -> Result { + self.transparent_transaction(|wdb,ext| -> anyhow::Result<_> { + let mut state=load(ext)?; state.durable=state.checkpoint_id.is_some(); + if let Some(scope)=&state.scope { + if wdb.block_fully_scanned()?.is_none_or(|b|u64::from(u32::from(b.block_height()))!=scope.anchor.height || b.block_hash().to_string()!=scope.anchor.hash) { state.complete=false; } + } + let total:u64=ext.query_row("SELECT COALESCE(SUM(n.value_zat),0) FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.mined_height IS NOT NULL AND NOT EXISTS(SELECT 1 FROM transparent_received_output_spends s WHERE s.transparent_received_output_id=n.id)",[],|r|r.get(0))?; + let mut status=state.status();status["confirmedZat"]=serde_json::json!(total.to_string());Ok(status) + }) + } + + pub fn transparent_pir_status_for(&self, source_id: &str) -> Result { + let status = self.transparent_pir_status()?; + if status.pointer("/scope/sourceId").and_then(|v| v.as_str()) != Some(source_id) { + return Ok( + serde_json::json!({"completion":"incomplete","spendingEnabled":false,"trust":"publisher-provided-confirmed-history"}), + ); + } + Ok(status) + } + + pub fn transparent_pir_checkpoint(&self) -> Result> { + self.transparent_transaction(|_, ext| -> anyhow::Result<_> { + Ok(load(ext)? + .continuation + .as_ref() + .map(serde_json::to_string) + .transpose()?) + }) + } +} diff --git a/crates/z-engine/src/native/wallet/transparent_pir_integration_tests.rs b/crates/z-engine/src/native/wallet/transparent_pir_integration_tests.rs new file mode 100644 index 0000000..84ae4e0 --- /dev/null +++ b/crates/z-engine/src/native/wallet/transparent_pir_integration_tests.rs @@ -0,0 +1,363 @@ +//! Real v12 PIR retrieval projected into both canonical wallets, checked against +//! independent serialized transaction ingestion. This fixture tests wallet data +//! semantics; it does not claim consensus-valid signatures or proof of work. +use super::*; +use crate::keys::account_from_mnemonic; +use crate::transparent_pir::acquisition::{ + Checkpoint, Limits, Operation, Pass, Reply, Scope, Step, Table, +}; +use crate::Network; +use base64::{engine::general_purpose::STANDARD, Engine}; +use transparent_events::{ReceiveEvent, SpendEvent, TransparentEvent, Txid}; +use transparent_filter::{BlockHash, ScriptBytes}; +use zcash_client_backend::proto::compact_formats::{CompactTx, CompactTxIn, TxOut}; + +#[allow(dead_code)] +#[path = "../../../../z-transparent-pir-v2/tests/fixture.rs"] +mod service; + +fn answer(server: &service::InProcess, operation: Operation) -> Reply { + let table = |t: Table| match t { + Table::Directory => "directory".into(), + Table::Pages { child } => format!("pages-{child}"), + }; + let (method, path, body) = match operation { + Operation::Init => ("GET", "/v2/shards/init".into(), vec![]), + Operation::Map => ("GET", "/v2/filters/shards".into(), vec![]), + Operation::Filter { shard_id } => ( + "GET", + format!("/v2/filters/shards/{shard_id}/filter"), + vec![], + ), + Operation::Manifest { shard_id, revision } => ( + "GET", + format!("/v2/shards/{shard_id}/revisions/{revision}/manifest"), + vec![], + ), + Operation::Setup { + shard_id, + revision, + table: t, + segment, + } => ( + "GET", + format!( + "/v2/shards/{shard_id}/revisions/{revision}/setup/{}/{segment}", + table(t) + ), + vec![], + ), + Operation::Query { + shard_id, + revision, + table: t, + body, + } => ( + "POST", + format!( + "/v2/shards/{shard_id}/revisions/{revision}/query/{}", + table(t) + ), + STANDARD.decode(body).unwrap(), + ), + }; + let response = server.answer( + axum::http::Request::builder() + .method(method) + .uri(path) + .body(body) + .unwrap(), + ); + assert_eq!( + response.status(), + 200, + "{}", + String::from_utf8_lossy(response.body()) + ); + Reply::Data { + body: STANDARD.encode(response.body()), + } +} +fn drive( + scope: Scope, + checkpoint: Option, + server: &service::InProcess, + mut apply: impl FnMut(&Checkpoint), +) -> Checkpoint { + let mut pass = Pass::new( + scope, + checkpoint, + Limits { + max_exchanges: 256, + ..Limits::default() + }, + ) + .unwrap(); + let mut latest = None; + for _ in 0..2000 { + match pass.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => pass.accept(request_id, answer(server, operation)).unwrap(), + Step::NeedChain { height, hash } => { + let expected = if height == 0 { + Network::Regtest.genesis_hash().into() + } else { + format!("{:02x}", height as u8).repeat(32) + }; + assert_eq!(hash, expected); + pass.accept_chain(height, expected).unwrap(); + } + Step::Checkpoint { checkpoint } => { + apply(&checkpoint); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + latest = Some(checkpoint); + } + Step::Stopped { .. } => return latest.unwrap(), + } + } + panic!("bounded fixture did not finish") +} +fn raw_transaction(previous: [u8; 32], index: u32, value: u64, script: &[u8]) -> Transaction { + let mut raw = vec![]; + raw.extend(1u32.to_le_bytes()); + raw.push(1); + raw.extend(previous); + raw.extend(index.to_le_bytes()); + raw.push(0); + raw.extend(u32::MAX.to_le_bytes()); + raw.push(1); + raw.extend(value.to_le_bytes()); + raw.push(u8::try_from(script.len()).unwrap()); + raw.extend(script); + raw.extend(0u32.to_le_bytes()); + let tx = Transaction::read(raw.as_slice(), BranchId::Sprout).unwrap(); + let mut canonical = vec![]; + tx.write(&mut canonical).unwrap(); + assert_eq!(canonical, raw); + tx +} +fn rows(wallet: &NativeWallet) -> Vec<(Vec, u32, Vec, u64, u32, Option>)> { + let conn = rusqlite::Connection::open(&wallet.paths.data_db).unwrap(); + let mut stmt=conn.prepare("SELECT t.txid,n.output_index,n.script,n.value_zat,t.mined_height,s.txid FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id LEFT JOIN transparent_received_output_spends l ON l.transparent_received_output_id=n.id LEFT JOIN transactions s ON s.id_tx=l.transaction_id ORDER BY t.txid,n.output_index").unwrap(); + stmt.query_map([], |r| { + Ok(( + r.get(0)?, + r.get(1)?, + r.get(2)?, + r.get(3)?, + r.get(4)?, + r.get(5)?, + )) + }) + .unwrap() + .collect::>>() + .unwrap() +} + +#[test] +fn transparent_pir_real_service_matches_independent_transaction_native_and_web_ingestion() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let root = tempfile::tempdir().unwrap(); + let reference_root = tempfile::tempdir().unwrap(); + let (fixture, wallet, reference_fixture, reference) = runtime.block_on(async { + let fixture = Fixture::new(root.path()).await; + let wallet = fixture.restore(root.path()).await; + wallet + .sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + let reference_fixture = Fixture::new(reference_root.path()).await; + let reference = reference_fixture.restore(reference_root.path()).await; + reference + .sync_with_wallet_source_reported_cancellable( + &reference_fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + (fixture, wallet, reference_fixture, reference) + }); + use zcash_keys::encoding::AddressCodec; + let address = transparent::address::TransparentAddress::decode( + &Network::Regtest, + &wallet.transparent_address().unwrap().unwrap(), + ) + .unwrap(); + let script: transparent::address::Script = address.script().into(); + let script = script.0 .0; + let funding = raw_transaction([9; 32], 0, 150000, &script); + let outsider: transparent::address::Script = + transparent::address::TransparentAddress::PublicKeyHash([7; 20]) + .script() + .into(); + let spending = raw_transaction(*funding.txid().as_ref(), 0, 140000, &outsider.0 .0); + // The source publication is derived from parsed serialized transactions. + let received = funding.transparent_bundle().unwrap().vout[0].clone(); + let consumed = spending.transparent_bundle().unwrap().vin[0].prevout(); + let receive = TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid(*funding.txid().as_ref()), + transaction_index: 1, + output_index: 0, + value: u64::from(received.value()), + coinbase: funding.transparent_bundle().unwrap().is_coinbase(), + }); + let spend = TransparentEvent::Spend(SpendEvent { + metadata: None, + height: 2, + spending_txid: Txid(*spending.txid().as_ref()), + transaction_index: 1, + input_index: 0, + spent_txid: Txid(*consumed.hash()), + spent_output_index: consumed.n(), + }); + let publication = tempfile::tempdir().unwrap(); + let genesis = BlockHash::from_display_hex(Network::Regtest.genesis_hash()).unwrap(); + service::publish_for_network( + publication.path(), + &[service::ShardSpec::first( + 0, + 2, + false, + vec![ + (ScriptBytes::new(script.clone()), receive), + (ScriptBytes::new(script.clone()), spend), + ], + )], + service::SEAL, + |h| { + if h == 0 { + genesis + } else { + BlockHash::from_internal_bytes([h as u8; 32]) + } + }, + vec![], + "regtest", + genesis, + ); + let server = service::InProcess::serve(publication.path()); + // Independent canonical reference consumes whole transactions, not source events. + let mut db = reference.open_db().unwrap(); + db.transactionally(|wdb| -> anyhow::Result<()> { + decrypt_and_store_transaction(&Network::Regtest, wdb, &funding, Some(1.into()))?; + decrypt_and_store_transaction(&Network::Regtest, wdb, &spending, Some(2.into()))?; + Ok(()) + }) + .unwrap(); + drop(db); + let mut native = wallet + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope: Scope = serde_json::from_str(&native.scope_json().unwrap()).unwrap(); + let checkpoint = drive(scope, None, &server, |cp| { + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut native, + &serde_json::to_string(cp).unwrap(), + ) + .unwrap(); + }); + assert!(checkpoint.complete); + assert_eq!(rows(&wallet), rows(&reference)); + assert_eq!(rows(&wallet).len(), 1); + assert_eq!(rows(&wallet)[0].3, 150000); + assert_eq!(rows(&wallet)[0].5, Some(spending.txid().as_ref().to_vec())); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "0" + ); + assert_eq!(wallet.history(20).unwrap().len(), 2); + let account = account_from_mnemonic(PUBLIC_SEED, Network::Regtest, 0).unwrap(); + let mut web = crate::web::WebWallet::from_account(account.clone(), 1).unwrap(); + let mut web_reference = crate::web::WebWallet::from_account(account, 1).unwrap(); + let mut first = block(1, Network::Regtest.genesis_hash()); + let mut second = block(2, Network::Regtest.genesis_hash()); + for b in [&mut first, &mut second] { + b.vtx.push(CompactTx { + index: 0, + txid: vec![0x66 + b.height as u8; 32], + ..Default::default() + }); + } + first.vtx.push(CompactTx { + index: 1, + txid: funding.txid().as_ref().to_vec(), + vout: vec![TxOut { + value: 150000, + script_pub_key: script.clone(), + }], + ..Default::default() + }); + second.vtx.push(CompactTx { + index: 1, + txid: spending.txid().as_ref().to_vec(), + vin: vec![CompactTxIn { + prevout_txid: funding.txid().as_ref().to_vec(), + prevout_index: 0, + }], + vout: vec![TxOut { + value: 140000, + script_pub_key: outsider.0 .0, + }], + ..Default::default() + }); + let compact = crate::web::encode_delimited(vec![first, second]); + web.apply_compact_blocks_blob(&compact).unwrap(); + web_reference.apply_compact_blocks_blob(&compact).unwrap(); + web_reference.apply_transparent_blocks(&compact).unwrap(); + let scope = + serde_json::from_str(&web.begin_transparent_pir(&"ab".repeat(32)).unwrap()).unwrap(); + drive(scope, None, &server, |cp| { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + let saved = web.to_snapshot().unwrap(); + assert!(!saved.is_empty()); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }); + let actual = web.history(20); + let expected = web_reference.history(20); + assert_eq!(actual.len(), 2); + assert_eq!(actual.len(), expected.len()); + for e in &actual { + let r = expected.iter().find(|r| r.txid == e.txid).unwrap(); + assert_eq!( + (e.transparent_received, e.transparent_spent), + (r.transparent_received, r.transparent_spent) + ); + } + assert_eq!(web.transparent_pir_status()["confirmedZat"], "0"); + let restored = crate::web::WebWallet::from_snapshot(&web.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.history(20).len(), 2); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "0"); + drop(native); + drop(wallet); + let reopened = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + assert_eq!(rows(&reopened), rows(&reference)); + assert!(!fixture.calls.lock().unwrap().iter().any(|m| matches!( + m.as_str(), + "GetTransaction" | "GetAddressUtxos" | "SendTransaction" + ))); + drop(server); + drop(fixture); + drop(reference_fixture); + drop(runtime); +} diff --git a/crates/z-engine/src/native/wallet/transparent_pir_source_tests.rs b/crates/z-engine/src/native/wallet/transparent_pir_source_tests.rs new file mode 100644 index 0000000..c09ef8f --- /dev/null +++ b/crates/z-engine/src/native/wallet/transparent_pir_source_tests.rs @@ -0,0 +1,346 @@ +use super::*; +use crate::transparent_pir::{TransparentPirCheckpoint, TransparentPirRecord, TransparentPirScope}; +use transparent_events::{ReceiveEvent, SpendEvent, TransparentEvent, Txid}; +use z_transparent_pir_v2::Coverage; + +fn checkpoint( + scope: TransparentPirScope, + script: &str, + events: Vec, + previous: Option, + id: u8, +) -> TransparentPirCheckpoint { + TransparentPirCheckpoint { + source_id: scope.source_id, + generation: scope.generation, + scope_id: scope.scope_id, + anchor: scope.anchor.clone(), + checkpoint_id: format!("{id:02x}").repeat(32), + previous_checkpoint_id: previous, + covered_through: scope.anchor.height, + complete: true, + pending_pages: 0, + unresolved_spends: 0, + records: events + .into_iter() + .map(|e| TransparentPirRecord { + script: script.into(), + event: crate::web::to_hex(&e.to_bytes()), + shard_id: 0, + revision: "ef".repeat(32), + }) + .collect(), + coverage: scope + .scripts + .into_iter() + .map(|s| Coverage { + script: s.script, + required_from: s.required_from, + through: scope.anchor.height, + revision: "ef".repeat(32), + }) + .collect(), + rewind_to: None, + continuation: String::new(), + } +} +fn receive() -> TransparentEvent { + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([0x11; 32]), + transaction_index: 1, + output_index: 0, + value: 150000, + coinbase: false, + }) +} +fn spend() -> TransparentEvent { + TransparentEvent::Spend(SpendEvent { + metadata: None, + height: 2, + spending_txid: Txid([0x22; 32]), + transaction_index: 1, + input_index: 0, + spent_txid: Txid([0x11; 32]), + spent_output_index: 0, + }) +} + +#[tokio::test] +async fn native_transparent_pir_canonical_receive_spend_reopen_and_cancel() { + use zcash_keys::encoding::AddressCodec; + let root = tempfile::tempdir().unwrap(); + let fixture = Fixture::new(root.path()).await; + let wallet = fixture.restore(root.path()).await; + wallet + .sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + let address = transparent::address::TransparentAddress::decode( + &wallet.network, + &wallet.transparent_address().unwrap().unwrap(), + ) + .unwrap(); + let script: transparent::address::Script = address.script().into(); + let script = crate::web::to_hex(&script.0 .0); + let cancel = Arc::new(RecoveryCancellation::new()); + let mut pass = wallet + .begin_transparent_pir_with_source(&fixture.source, &"ab".repeat(32), cancel) + .unwrap(); + let scope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let mut c = checkpoint(scope, &script, vec![receive()], None, 1); + c.complete = false; + c.pending_pages = 1; + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&c).unwrap(), + ) + .unwrap(); + let conn = rusqlite::Connection::open(&wallet.paths.data_db).unwrap(); + let (value,raw):(u64,Option>)=conn.query_row("SELECT n.value_zat,t.raw FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid=?1",[[0x11u8;32].as_slice()],|r|Ok((r.get(0)?,r.get(1)?))).unwrap(); + assert_eq!(value, 150000); + assert!(raw.is_none()); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "150000" + ); + conn.execute( + "UPDATE transparent_received_outputs SET lock_expiry_height=100 WHERE value_zat=150000", + [], + ) + .unwrap(); + // Reapplying identical evidence must not erase a reservation. + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&c).unwrap(), + ) + .unwrap(); + assert_eq!( + conn.query_row( + "SELECT lock_expiry_height FROM transparent_received_outputs WHERE value_zat=150000", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 100 + ); + let scope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let c2 = checkpoint( + scope, + &script, + vec![receive(), spend()], + Some(c.checkpoint_id.clone()), + 2, + ); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&c2).unwrap(), + ) + .unwrap(); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "0" + ); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM transparent_received_output_spends", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 1 + ); + assert_eq!(wallet.history(20).unwrap().len(), 2); + assert_eq!( + wallet.transparent_pir_status().unwrap()["completion"], + "incomplete" + ); + drop(pass); + drop(wallet); + let reopened = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["confirmedZat"], + "0" + ); + assert_eq!(reopened.history(20).unwrap().len(), 2); + let mut recut = reopened + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&recut.scope_json().unwrap()).unwrap(); + let mut c3 = checkpoint( + scope, + &script, + vec![receive()], + Some(c2.checkpoint_id.clone()), + 3, + ); + c3.rewind_to = Some(crate::transparent_pir::TransparentPirAnchor { + height: 1, + hash: "01".repeat(32), + }); + reopened + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut recut, + &serde_json::to_string(&c3).unwrap(), + ) + .unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["confirmedZat"], + "150000" + ); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM transparent_received_output_spends", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 0 + ); + assert_eq!( + conn.query_row( + "SELECT lock_expiry_height FROM transparent_received_outputs WHERE value_zat=150000", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 100 + ); + drop(recut); + let token = Arc::new(RecoveryCancellation::new()); + let mut pass = reopened + .begin_transparent_pir_with_source(&fixture.source, &"ab".repeat(32), token.clone()) + .unwrap(); + let scope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let c4 = checkpoint(scope, &script, vec![receive()], Some(c3.checkpoint_id), 4); + token.cancel(); + assert!(reopened + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&c4).unwrap() + ) + .is_err()); + assert_eq!( + reopened + .transparent_pir_checkpoint() + .unwrap() + .map(|s| serde_json::from_str::(&s) + .unwrap() + .checkpoint_id), + Some("03".repeat(32)) + ); + drop(pass); + let mut mined = reopened + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&mined.scope_json().unwrap()).unwrap(); + let c5 = checkpoint( + scope, + &script, + vec![receive(), spend()], + Some("03".repeat(32)), + 5, + ); + reopened + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut mined, + &serde_json::to_string(&c5).unwrap(), + ) + .unwrap(); + drop(mined); + reopened + .open_db() + .unwrap() + .truncate_to_height(1.into()) + .unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["completion"], + "incomplete" + ); + let mut after_reorg = reopened + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope: TransparentPirScope = + serde_json::from_str(&after_reorg.scope_json().unwrap()).unwrap(); + assert_eq!(scope.anchor.height, 1); + let mut partial = checkpoint(scope.clone(), &script, vec![], None, 6); + partial.complete = false; + partial.pending_pages = 1; + reopened + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut after_reorg, + &serde_json::to_string(&partial).unwrap(), + ) + .unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["completion"], + "incomplete" + ); + let c6 = checkpoint( + scope, + &script, + vec![receive()], + Some(partial.checkpoint_id), + 7, + ); + reopened + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut after_reorg, + &serde_json::to_string(&c6).unwrap(), + ) + .unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["confirmedZat"], + "150000" + ); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM transparent_received_output_spends", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 0 + ); + assert_eq!( + conn.query_row( + "SELECT lock_expiry_height FROM transparent_received_outputs WHERE value_zat=150000", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 100 + ); + assert!(!fixture.calls.lock().unwrap().iter().any(|m| matches!( + m.as_str(), + "GetTransaction" | "GetAddressUtxos" | "SendTransaction" + ))); +} diff --git a/crates/z-engine/src/transparent_pir.rs b/crates/z-engine/src/transparent_pir.rs new file mode 100644 index 0000000..1172360 --- /dev/null +++ b/crates/z-engine/src/transparent_pir.rs @@ -0,0 +1,335 @@ +//! Publisher-accepted transparent discovery metadata. Canonical wallet rows own value. +use crate::error::{EngineError, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use transparent_events::TransparentEvent; + +pub(crate) fn refused(code: &str) -> EngineError { + EngineError::Message(code.into()) +} + +pub use z_transparent_pir_v2 as acquisition; +pub use z_transparent_pir_v2::{ + Anchor as TransparentPirAnchor, Checkpoint as TransparentPirCheckpoint, + Record as TransparentPirRecord, Scope as TransparentPirScope, Script as TransparentPirScript, +}; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ProjectionState { + pub scope: Option, + #[serde(default)] + pub generation: u64, + #[serde(default)] + pub wallet_cursor: Option, + #[serde(default)] + pub bound_source: Option, + #[serde(default)] + pub rewind_floor: Option, + pub checkpoint_id: Option, + pub checkpoint_digest: Option, + pub continuation: Option, + pub covered_through: u64, + pub complete: bool, + pub pending_pages: u64, + pub unresolved_spends: u64, + /// Identities and digests only. No second copy of wallet values or balance. + pub records: BTreeMap, + #[serde(default)] + pub projected_outputs: BTreeSet, + #[serde(default)] + pub projected_spends: BTreeSet, + #[serde(default)] + pub projected_transactions: BTreeSet, + #[serde(skip)] + pub durable: bool, +} + +pub(crate) fn hash(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +pub(crate) fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +pub(crate) fn parse(json: &str) -> Result { + if json.len() > 128 * 1024 * 1024 { + return Err(refused("transparent_pir_size_limit")); + } + serde_json::from_str(json).map_err(|_| refused("transparent_pir_input_invalid")) +} + +impl ProjectionState { + pub fn begin( + &mut self, + source: &str, + anchor: TransparentPirAnchor, + mut scripts: Vec, + ) -> Result { + if !hash(source) || !hash(&anchor.hash) || scripts.is_empty() || scripts.len() > 4096 { + return Err(refused("transparent_pir_scope_invalid")); + } + if self + .bound_source + .as_deref() + .or_else(|| self.scope.as_ref().map(|s| s.source_id.as_str())) + .is_some_and(|old| old != source) + { + return Err(refused("transparent_pir_source_changed")); + } + scripts.sort_by(|a, b| a.script.cmp(&b.script)); + scripts.dedup(); + if scripts.iter().any(|s| { + s.required_from > anchor.height + || crate::web::from_hex(&s.script) + .ok() + .is_none_or(|b| b.is_empty() || b.len() > 40) + }) { + return Err(refused("transparent_pir_scope_invalid")); + } + let scope_id = digest( + &serde_json::to_vec(&scripts).map_err(|_| refused("transparent_pir_scope_invalid"))?, + ); + if let Some(old) = &self.scope { + if old.source_id == source && old.anchor == anchor && old.scope_id == scope_id { + return Ok(old.clone()); + } + // A changed chain must first pass through the wallet's canonical rewind. + if anchor.height < old.anchor.height + || (anchor.height == old.anchor.height && anchor.hash != old.anchor.hash) + { + return Err(refused("transparent_pir_rewind_required")); + } + } + let generation = self + .generation + .checked_add(1) + .ok_or_else(|| refused("transparent_pir_generation_exhausted"))?; + let same_source = self.bound_source.as_deref() == Some(source) + || self.scope.as_ref().is_some_and(|s| s.source_id == source); + self.bound_source = Some(source.into()); + self.generation = generation; + let scope = TransparentPirScope { + source_id: source.into(), + generation, + scope_id, + accepted_headers: vec![anchor.clone()], + anchor, + scripts, + }; + self.scope = Some(scope.clone()); + if !same_source { + self.checkpoint_id = None; + self.checkpoint_digest = None; + self.continuation = None; + self.covered_through = 0; + self.projected_outputs.clear(); + self.projected_spends.clear(); + self.projected_transactions.clear(); + } + self.complete = false; + self.durable = false; + Ok(scope) + } + + pub fn validate( + &self, + checkpoint: &TransparentPirCheckpoint, + ) -> Result, TransparentEvent)>> { + let scope = self + .scope + .as_ref() + .ok_or_else(|| refused("transparent_pir_consent_required"))?; + if checkpoint.source_id != scope.source_id + || checkpoint.generation != scope.generation + || checkpoint.scope_id != scope.scope_id + || checkpoint.anchor != scope.anchor + { + return Err(refused("transparent_pir_context_changed")); + } + let bytes = checkpoint_payload(checkpoint)?; + if self.checkpoint_id.as_ref() == Some(&checkpoint.checkpoint_id) { + return if self.checkpoint_digest.as_ref() == Some(&digest(&bytes)) { + Ok(Vec::new()) + } else { + Err(refused("transparent_pir_checkpoint_conflict")) + }; + } + if !hash(&checkpoint.checkpoint_id) + || checkpoint.previous_checkpoint_id != self.checkpoint_id + || checkpoint.covered_through < self.covered_through + || checkpoint.covered_through > scope.anchor.height + || checkpoint.records.len() > 65_536 + || (checkpoint.complete + && (checkpoint.covered_through != scope.anchor.height + || checkpoint.pending_pages != 0 + || checkpoint.unresolved_spends != 0)) + { + return Err(refused("transparent_pir_checkpoint_invalid")); + } + if checkpoint.complete + && scope.scripts.iter().any(|s| { + !checkpoint.coverage.iter().any(|c| { + c.script == s.script + && c.required_from <= s.required_from + && c.through >= scope.anchor.height + && hash(&c.revision) + }) + }) + { + return Err(refused("transparent_pir_coverage_incomplete")); + } + let mut decoded = Vec::new(); + let mut identities = BTreeMap::new(); + let mut received_totals = BTreeMap::<[u8; 32], u64>::new(); + for record in &checkpoint.records { + if !hash(&record.revision) { + return Err(refused("transparent_pir_revision_invalid")); + } + let entry = scope + .scripts + .iter() + .find(|s| s.script == record.script) + .ok_or_else(|| refused("transparent_pir_script_not_owned"))?; + let script = crate::web::from_hex(&record.script) + .map_err(|_| refused("transparent_pir_record_invalid"))?; + let bytes = crate::web::from_hex(&record.event) + .map_err(|_| refused("transparent_pir_record_invalid"))?; + let event = TransparentEvent::from_bytes(&bytes) + .map_err(|_| refused("transparent_pir_record_invalid"))?; + if event.to_bytes() != bytes + || u64::from(event_height(event)) < entry.required_from + || u64::from(event_height(event)) > scope.anchor.height + { + return Err(refused("transparent_pir_record_invalid")); + } + if let TransparentEvent::Receive(receive) = event { + let total = received_totals.entry(receive.txid.0).or_default(); + *total = total + .checked_add(receive.value) + .ok_or_else(|| refused("transparent_pir_amount_invalid"))?; + zcash_protocol::value::Zatoshis::from_u64(*total) + .map_err(|_| refused("transparent_pir_amount_invalid"))?; + } + let key = event_key(event); + let fact = (event_height(event), digest(&bytes)); + if self.records.get(&key).is_some_and(|old| *old != fact) + || identities.insert(key, fact).is_some() + { + return Err(refused("transparent_pir_record_conflict")); + } + decoded.push((script, event)); + } + decoded.sort_by_key(|(_, e)| match e { + TransparentEvent::Receive(e) => (e.height, e.transaction_index, 0, e.output_index), + TransparentEvent::Spend(e) => (e.height, e.transaction_index, 1, e.input_index), + }); + Ok(decoded) + } + + pub fn prepare_rewind(&mut self, height: u32) { + self.records.retain(|_, (h, _)| *h <= height); + self.covered_through = self.covered_through.min(u64::from(height)); + self.complete = false; + } + + pub fn stage( + &mut self, + checkpoint: &TransparentPirCheckpoint, + records: &[(Vec, TransparentEvent)], + ) -> Result<()> { + for (_, event) in records { + self.records.insert( + event_key(*event), + (event_height(*event), digest(&event.to_bytes())), + ); + } + self.rewind_floor = None; + self.checkpoint_id = Some(checkpoint.checkpoint_id.clone()); + self.checkpoint_digest = Some(digest(&checkpoint_payload(checkpoint)?)); + let mut saved = checkpoint.clone(); + saved.records.clear(); + self.continuation = Some(saved); + self.covered_through = checkpoint.covered_through; + self.complete = checkpoint.complete; + self.pending_pages = checkpoint.pending_pages; + self.unresolved_spends = checkpoint.unresolved_spends; + self.durable = false; + Ok(()) + } + + pub fn acknowledge(&mut self, id: &str) -> Result<()> { + if self.checkpoint_id.as_deref() != Some(id) { + return Err(refused("transparent_pir_checkpoint_changed")); + } + self.durable = true; + Ok(()) + } + + pub fn rewind(&mut self, height: u32) { + if self.bound_source.is_none() && self.scope.is_none() { + return; + } + self.rewind_floor = Some(self.rewind_floor.map_or(height, |old| old.min(height))); + self.complete = false; + self.covered_through = self.covered_through.min(u64::from(height)); + self.checkpoint_id = None; + self.checkpoint_digest = None; + self.continuation = None; + self.scope = None; + self.durable = false; + } + + pub fn status(&self) -> serde_json::Value { + serde_json::json!({ "trust":"publisher-provided-confirmed-history", "spendingEnabled":false, + "completion":if self.complete && self.durable { "complete-for-enrolled-scope" } else if self.complete { "checkpoint-pending" } else { "incomplete" }, + "scope": self.scope.as_ref().map(|s|serde_json::json!({"sourceId":s.source_id,"scopeId":s.scope_id,"generation":s.generation,"anchor":s.anchor,"scriptCount":s.scripts.len()})), "checkpointId":self.checkpoint_id, "coveredThrough":self.covered_through, + "pendingPages":self.pending_pages, "unresolvedSpends":self.unresolved_spends }) + } +} + +pub(crate) fn event_height(event: TransparentEvent) -> u32 { + match event { + TransparentEvent::Receive(e) => e.height, + TransparentEvent::Spend(e) => e.height, + } +} +pub(crate) fn event_key(event: TransparentEvent) -> String { + match event { + TransparentEvent::Receive(e) => { + format!("r:{}:{}", crate::web::to_hex(&e.txid.0), e.output_index) + } + TransparentEvent::Spend(e) => format!( + "s:{}:{}:{}:{}", + crate::web::to_hex(&e.spending_txid.0), + e.input_index, + crate::web::to_hex(&e.spent_txid.0), + e.spent_output_index + ), + } +} + +fn checkpoint_payload(checkpoint: &TransparentPirCheckpoint) -> Result> { + let mut normalized = checkpoint.clone(); + normalized.previous_checkpoint_id = None; + serde_json::to_vec(&normalized).map_err(|_| refused("transparent_pir_input_invalid")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn never_enrolled_rewind_preserves_exact_serialized_state() { + let mut state = ProjectionState::default(); + let saved = serde_json::to_vec(&state).unwrap(); + state.rewind(1); + assert_eq!(serde_json::to_vec(&state).unwrap(), saved); + assert!(state.rewind_floor.is_none()); + } +} diff --git a/crates/z-engine/src/web/prove.rs b/crates/z-engine/src/web/prove.rs index 25ac526..dae042c 100644 --- a/crates/z-engine/src/web/prove.rs +++ b/crates/z-engine/src/web/prove.rs @@ -148,6 +148,12 @@ pub(super) fn orchard_spend_policy() -> SpendPolicy { impl WebWallet { pub fn prove_shield(&mut self, mnemonic: &str, threshold_zat: u64) -> Result> { + #[cfg(feature = "transparent-pir")] + if self.transparent_pir.scope.is_some() { + return Err(EngineError::Message( + "transparent_pir_shield_unavailable".into(), + )); + } #[cfg(not(feature = "transparent-inputs"))] { let _ = (mnemonic, threshold_zat); diff --git a/crates/z-engine/src/web/public_data.rs b/crates/z-engine/src/web/public_data.rs index 4e12d76..2ece8fe 100644 --- a/crates/z-engine/src/web/public_data.rs +++ b/crates/z-engine/src/web/public_data.rs @@ -447,7 +447,7 @@ impl WebWallet { } #[cfg(feature = "transparent-inputs")] - fn public_transparent_scripts(&mut self) -> Result, String)>> { + pub(crate) fn public_transparent_scripts(&mut self) -> Result, String)>> { use transparent::{ address::{Script, TransparentAddress}, keys::{NonHardenedChildIndex, TransparentKeyScope}, diff --git a/crates/z-engine/src/web/store.rs b/crates/z-engine/src/web/store.rs index 8f8494f..9e8b2c6 100644 --- a/crates/z-engine/src/web/store.rs +++ b/crates/z-engine/src/web/store.rs @@ -1,4 +1,7 @@ //! Versioned in-memory wallet snapshot (IndexedDB blob on the JS side). +#[cfg(feature = "transparent-pir")] +#[path = "transparent_pir.rs"] +mod transparent_pir_impl; use super::history; use super::live_trees::{self, StoredLiveTrees}; @@ -345,6 +348,9 @@ struct Snapshot { transparent_scan: Option, #[serde(default)] transparent_scan_required: bool, + #[cfg(feature = "transparent-pir")] + #[serde(default)] + transparent_pir: crate::transparent_pir::ProjectionState, #[serde(default)] memo_scan: Option, #[serde(default)] @@ -420,6 +426,8 @@ struct SnapshotRef<'a> { utxos: &'a [StoredUtxo], transparent_scan: &'a Option, transparent_scan_required: bool, + #[cfg(feature = "transparent-pir")] + transparent_pir: &'a crate::transparent_pir::ProjectionState, memo_scan: &'a Option, block_hashes: &'a BTreeMap, view_only: bool, @@ -524,6 +532,8 @@ pub struct WebWallet { pub(crate) utxos: Vec, pub(crate) transparent_scan: Option, pub(crate) transparent_scan_required: bool, + #[cfg(feature = "transparent-pir")] + pub(crate) transparent_pir: crate::transparent_pir::ProjectionState, pub(crate) memo_scan: Option, pub(crate) block_hashes: BTreeMap, view_only: bool, @@ -601,6 +611,8 @@ impl WebWallet { utxos: Vec::new(), transparent_scan: None, transparent_scan_required: false, + #[cfg(feature = "transparent-pir")] + transparent_pir: Default::default(), memo_scan: None, block_hashes: BTreeMap::new(), view_only: false, @@ -995,6 +1007,8 @@ impl WebWallet { utxos: &self.utxos, transparent_scan: &self.transparent_scan, transparent_scan_required: self.transparent_scan_required, + #[cfg(feature = "transparent-pir")] + transparent_pir: &self.transparent_pir, memo_scan: &self.memo_scan, block_hashes: &self.block_hashes, view_only: self.view_only, @@ -1072,6 +1086,8 @@ impl WebWallet { utxos: snap.utxos, transparent_scan: snap.transparent_scan, transparent_scan_required: snap.transparent_scan_required, + #[cfg(feature = "transparent-pir")] + transparent_pir: snap.transparent_pir, memo_scan: snap.memo_scan, block_hashes: snap.block_hashes, view_only: snap.view_only, @@ -1163,6 +1179,10 @@ impl WebWallet { if let Some(live) = live_trees { w.restore_live_trees(&live); } + #[cfg(feature = "transparent-pir")] + { + w.transparent_pir.durable = w.transparent_pir.checkpoint_id.is_some(); + } w.recompute_pool_fields(); Ok(w) } @@ -1397,6 +1417,14 @@ impl WebWallet { target: zcash_client_backend::data_api::wallet::TargetHeight, policy: &zcash_client_backend::data_api::wallet::ConfirmationsPolicy, ) -> bool { + #[cfg(feature = "transparent-pir")] + if self + .transparent_pir + .records + .contains_key(&format!("r:{}:{}", u.txid, u.index)) + { + return false; + } if self.transparent_scan_required && self .transparent_scan @@ -1417,6 +1445,15 @@ impl WebWallet { tip > u.height && tip.saturating_sub(u.height) >= need } + fn transparent_pir_snapshot_status(&self) -> serde_json::Value { + #[cfg(feature = "transparent-pir")] + { + return self.transparent_pir_status(); + } + #[cfg(not(feature = "transparent-pir"))] + serde_json::Value::Null + } + pub fn wallet_snapshot(&self, server: &str) -> serde_json::Value { let b = self.balance(); let zip = zip321_uri(&self.unified_address, None).unwrap_or_default(); @@ -1433,6 +1470,7 @@ impl WebWallet { "transparentAddress": self.transparent_address, "zip321": zip, "scannedHeight": self.scanned_height, + "transparentPir": self.transparent_pir_snapshot_status(), "transparentScanHeight": self.transparent_scan.as_ref().map(|s| s.height), "transparentScanComplete": self.transparent_scan.as_ref().is_some_and(|s| s.height == self.scanned_height), "memoScanHeight": self.memo_scan.as_ref().map(|s| s.height), @@ -2040,6 +2078,8 @@ impl WebWallet { self.restore_next_after_rewind(birthday_bound); } self.rewind_public_coverage(); + #[cfg(feature = "transparent-pir")] + self.transparent_pir.rewind(keep); self.rebuild_trees()?; Ok(self.scanned_height) } diff --git a/crates/z-engine/src/web/transparent_pir.rs b/crates/z-engine/src/web/transparent_pir.rs new file mode 100644 index 0000000..10ccc81 --- /dev/null +++ b/crates/z-engine/src/web/transparent_pir.rs @@ -0,0 +1,620 @@ +use super::*; +use crate::transparent_pir::{ + parse, refused, TransparentPirAnchor, TransparentPirCheckpoint, TransparentPirScript, +}; +use transparent_events::TransparentEvent; + +impl WebWallet { + /// The caller has accepted this exact transparent publisher profile. All scripts + /// and the target are obtained from this wallet; no host-provided amount is used. + pub fn begin_transparent_pir(&mut self, source_id: &str) -> Result { + let height = self.scanned_height; + let mut hash = super::super::from_hex( + self.hash_at(height) + .ok_or_else(|| refused("transparent_pir_anchor_unavailable"))?, + ) + .map_err(|_| refused("transparent_pir_anchor_unavailable"))?; + hash.reverse(); + let anchor = TransparentPirAnchor { + height: u64::from(height), + hash: super::super::to_hex(&hash), + }; + let scripts = self + .public_transparent_scripts()? + .into_iter() + .map(|(script, _)| TransparentPirScript { + script: super::super::to_hex(&script), + required_from: 0, + }) + .collect(); + let mut scope = self.transparent_pir.begin(source_id, anchor, scripts)?; + scope.accepted_headers.retain(|h| h.height != 0); + scope.accepted_headers.push(TransparentPirAnchor { + height: 0, + hash: self.network.genesis_hash().into(), + }); + self.transparent_pir.scope = Some(scope.clone()); + self.transparent_pir.wallet_cursor = Some(self.next_diversifier); + serde_json::to_string(&scope).map_err(|_| refused("transparent_pir_scope_invalid")) + } + + pub fn transparent_pir_checkpoint(&self) -> Option { + self.transparent_pir + .continuation + .as_ref() + .and_then(|c| serde_json::to_string(c).ok()) + } + + pub fn transparent_pir_status(&self) -> serde_json::Value { + let mut state = self.transparent_pir.clone(); + if state + .scope + .as_ref() + .is_some_and(|scope| scope.anchor.height != u64::from(self.scanned_height)) + { + state.complete = false; + } + if state.wallet_cursor != Some(self.next_diversifier) { + state.complete = false; + } + if let Some(scope) = &state.scope { + let known = u32::try_from(scope.anchor.height) + .ok() + .and_then(|h| self.hash_at(h)) + .and_then(|h| super::super::from_hex(h).ok()) + .map(|mut h| { + h.reverse(); + super::super::to_hex(&h) + }); + if known.as_deref() != Some(&scope.anchor.hash) { + state.complete = false; + } + } + let mut status = state.status(); + let total: u64 = self + .utxos + .iter() + .filter(|u| { + !u.spent + && self + .transparent_pir + .records + .contains_key(&format!("r:{}:{}", u.txid, u.index)) + }) + .map(|u| u.value_zat) + .sum(); + status["confirmedZat"] = serde_json::json!(total.to_string()); + status + } + + pub fn apply_transparent_pir_checkpoint(&mut self, json: &str) -> Result { + let checkpoint: TransparentPirCheckpoint = parse(json)?; + let mut state = self.transparent_pir.clone(); + let old_records = state.records.clone(); + let retirement_floor = state + .rewind_floor + .into_iter() + .chain(checkpoint.rewind_to.as_ref().map(|a| a.height as u32)) + .min(); + if let Some(floor) = retirement_floor { + state.prepare_rewind(floor); + } + if let Some(ancestor) = &checkpoint.rewind_to { + let height = u32::try_from(ancestor.height) + .map_err(|_| refused("transparent_pir_anchor_invalid"))?; + let known = if height == 0 { + Some(self.network.genesis_hash().to_string()) + } else { + self.hash_at(height) + .and_then(|h| super::super::from_hex(h).ok()) + .map(|mut h| { + h.reverse(); + super::super::to_hex(&h) + }) + }; + if known.as_deref() != Some(&ancestor.hash) { + return Err(refused("transparent_pir_rewind_anchor_unknown")); + } + state.prepare_rewind(height); + } + let records = state.validate(&checkpoint)?; + let height = u32::try_from(checkpoint.anchor.height) + .map_err(|_| refused("transparent_pir_anchor_invalid"))?; + let mut hash = super::super::from_hex(&checkpoint.anchor.hash) + .map_err(|_| refused("transparent_pir_anchor_invalid"))?; + hash.reverse(); + if self.hash_at(height) != Some(super::super::to_hex(&hash).as_str()) + || self.scanned_height != height + { + return Err(refused("transparent_pir_chain_changed")); + } + if self.transparent_pir.checkpoint_id.as_ref() == Some(&checkpoint.checkpoint_id) { + return serde_json::to_string(&self.transparent_pir_status()) + .map_err(|_| refused("transparent_pir_status_invalid")); + } + if retirement_floor.is_some() && !checkpoint.complete { + state.records = old_records; + state.stage(&checkpoint, &[])?; + state.rewind_floor = retirement_floor; + self.transparent_pir = state; + return serde_json::to_string(&self.transparent_pir_status()) + .map_err(|_| refused("transparent_pir_status_invalid")); + } + let scripts = self.public_transparent_scripts()?; + // Stage all canonical effects on clones. A malformed late record cannot + // partially mutate outputs, pending reservations or transaction history. + let mut utxos = self.utxos.clone(); + let mut txs = self.txs.clone(); + let mut next_diversifier = self.next_diversifier; + if let Some(floor) = retirement_floor { + let incoming: std::collections::BTreeMap<_, _> = records + .iter() + .map(|(_, e)| { + ( + crate::transparent_pir::event_key(*e), + crate::transparent_pir::digest(&e.to_bytes()), + ) + }) + .collect(); + let retired: Vec<_> = old_records + .iter() + .filter(|(key, (h, d))| *h > floor && incoming.get(*key) != Some(d)) + .map(|(k, _)| k.clone()) + .collect(); + for key in &retired { + let fields: Vec<_> = key.split(':').collect(); + if fields.first() == Some(&"s") && state.projected_spends.remove(key) { + let spending = Self::txid_from_hex(fields[1])?; + if self.pending_txs.contains_key(&spending) { + return Err(refused("transparent_pir_pending_reconciliation_required")); + } + let index: u32 = fields[4] + .parse() + .map_err(|_| refused("transparent_pir_state_invalid"))?; + if let Some(u) = utxos.iter_mut().find(|u| { + u.txid == fields[3] + && u.index == index + && u.spent_in.as_deref() == Some(fields[1]) + }) { + u.spent = false; + u.spent_in = None; + } + } + } + for key in &retired { + let fields: Vec<_> = key.split(':').collect(); + if fields.first() == Some(&"r") && state.projected_outputs.remove(key) { + let index: u32 = fields[2] + .parse() + .map_err(|_| refused("transparent_pir_state_invalid"))?; + if self + .output_locks + .iter() + .any(|l| l.txid == fields[1] && l.index == index && l.pool == "transparent") + || utxos.iter().any(|u| { + u.txid == fields[1] + && u.index == index + && u.spent_in.as_ref().is_some_and(|t| { + Self::txid_from_hex(t) + .ok() + .is_some_and(|id| self.pending_txs.contains_key(&id)) + }) + }) + { + return Err(refused("transparent_pir_pending_reconciliation_required")); + } + utxos.retain(|u| u.txid != fields[1] || u.index != index); + } + } + for id in &state.projected_transactions { + if !self.pending_txs.contains_key(&Self::txid_from_hex(id)?) { + if let Some(agg) = txs.get_mut(id) { + if agg.mined_height.is_some_and(|h| h > floor) + && !agg.has_shielded_activity() + { + agg.mined_height = None; + agg.transparent_received = 0; + agg.transparent_spent = 0; + agg.received_zat = 0; + agg.spent_zat = 0; + } + } + } + } + txs.retain(|id, agg| { + !state.projected_transactions.contains(id) + || agg.mined_height.is_some() + || agg.has_shielded_activity() + || Self::txid_from_hex(id) + .ok() + .is_some_and(|id| self.pending_txs.contains_key(&id)) + }); + } + for (script, event) in &records { + let (position, (_, address)) = scripts + .iter() + .enumerate() + .find(|(_, (s, _))| s == script) + .ok_or_else(|| refused("transparent_pir_scope_changed"))?; + match event { + TransparentEvent::Receive(e) => { + let txid = super::super::to_hex(&e.txid.0); + if let Some(u) = utxos + .iter_mut() + .find(|u| u.txid == txid && u.index == e.output_index) + { + if u.value_zat != e.value + || u.script != super::super::to_hex(script) + || (u.height != 0 && u.height != e.height) + { + return Err(refused("transparent_pir_output_conflict")); + } + u.height = e.height; + u.coinbase = e.coinbase; + } else { + state + .projected_outputs + .insert(crate::transparent_pir::event_key(*event)); + utxos.push(StoredUtxo { + txid: txid.clone(), + index: e.output_index, + script: super::super::to_hex(script), + value_zat: e.value, + height: e.height, + address: address.clone(), + coinbase: e.coinbase, + spent: false, + spent_in: None, + }); + } + if !txs.contains_key(&txid) { + state.projected_transactions.insert(txid.clone()); + } + let agg = txs.entry(txid).or_default(); + if agg.mined_height.is_some_and(|h| h != e.height) { + return Err(refused("transparent_pir_transaction_conflict")); + } + agg.mined_height = Some(e.height); + next_diversifier = next_diversifier.max( + u32::try_from(position / 2 + 1) + .map_err(|_| refused("transparent_pir_scope_limit"))?, + ); + } + TransparentEvent::Spend(e) => { + let txid = super::super::to_hex(&e.spending_txid.0); + let previous = super::super::to_hex(&e.spent_txid.0); + if let Some(u) = utxos + .iter_mut() + .find(|u| u.txid == previous && u.index == e.spent_output_index) + { + if u.script != super::super::to_hex(script) + || (u.spent && u.spent_in.as_ref().is_some_and(|known| known != &txid)) + { + return Err(refused("transparent_pir_spend_conflict")); + } + if !u.spent { + state + .projected_spends + .insert(crate::transparent_pir::event_key(*event)); + } + u.spent = true; + u.spent_in = Some(txid.clone()); + } else if checkpoint.complete { + return Err(refused("transparent_pir_unresolved_spend")); + } + if !txs.contains_key(&txid) { + state.projected_transactions.insert(txid.clone()); + } + let agg = txs.entry(txid).or_default(); + if agg.mined_height.is_some_and(|h| h != e.height) { + return Err(refused("transparent_pir_transaction_conflict")); + } + agg.mined_height = Some(e.height); + } + } + } + let mut unspent = 0u64; + let mut spends = std::collections::BTreeMap::<&str, u64>::new(); + for u in &utxos { + let total = if let Some(spender) = u.spent_in.as_deref() { + spends.entry(spender).or_default() + } else if !u.spent { + &mut unspent + } else { + continue; + }; + *total = total + .checked_add(u.value_zat) + .ok_or_else(|| refused("transparent_pir_amount_invalid"))?; + zcash_protocol::value::Zatoshis::from_u64(*total) + .map_err(|_| refused("transparent_pir_amount_invalid"))?; + } + state.stage(&checkpoint, &records)?; + // A received gap address expands inventory, so this checkpoint cannot + // assert completeness for the new scripts until a further pass covers them. + if next_diversifier != self.next_diversifier { + state.complete = false; + } + self.utxos = utxos; + self.txs = txs; + self.next_diversifier = next_diversifier; + self.transparent_pir = state; + self.recompute_pool_fields(); + serde_json::to_string(&self.transparent_pir_status()) + .map_err(|_| refused("transparent_pir_status_invalid")) + } + + pub fn acknowledge_transparent_pir_checkpoint(&mut self, id: &str) -> Result<()> { + self.transparent_pir.acknowledge(id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::keys::{account_from_mnemonic, REGTEST_FAUCET_MNEMONIC}; + use crate::transparent_pir::{TransparentPirRecord, TransparentPirScope}; + use transparent_events::{ReceiveEvent, SpendEvent, Txid}; + use z_transparent_pir_v2::Coverage; + + pub(super) fn wallet() -> WebWallet { + let mut w = WebWallet::from_account( + account_from_mnemonic(REGTEST_FAUCET_MNEMONIC, Network::Regtest, 0).unwrap(), + 1, + ) + .unwrap(); + w.scanned_height = 3; + w.block_hashes.insert(3, "03".repeat(32)); + w + } + pub(super) fn checkpoint( + w: &mut WebWallet, + events: Vec, + ) -> TransparentPirCheckpoint { + let scope: TransparentPirScope = + serde_json::from_str(&w.begin_transparent_pir(&"ab".repeat(32)).unwrap()).unwrap(); + let script = super::super::super::to_hex(&w.public_transparent_scripts().unwrap()[0].0); + TransparentPirCheckpoint { + source_id: scope.source_id, + generation: scope.generation, + scope_id: scope.scope_id, + anchor: scope.anchor, + checkpoint_id: "cd".repeat(32), + previous_checkpoint_id: None, + covered_through: 3, + complete: true, + pending_pages: 0, + unresolved_spends: 0, + records: events + .into_iter() + .map(|e| TransparentPirRecord { + script: script.clone(), + event: super::super::super::to_hex(&e.to_bytes()), + shard_id: 0, + revision: "ef".repeat(32), + }) + .collect(), + coverage: scope + .scripts + .into_iter() + .map(|s| Coverage { + script: s.script, + required_from: s.required_from, + through: 3, + revision: "ef".repeat(32), + }) + .collect(), + rewind_to: None, + continuation: String::new(), + } + } + pub(super) fn receive() -> TransparentEvent { + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([1; 32]), + transaction_index: 1, + output_index: 0, + value: 150000, + coinbase: false, + }) + } + pub(super) fn spend() -> TransparentEvent { + TransparentEvent::Spend(SpendEvent { + metadata: None, + height: 2, + spending_txid: Txid([2; 32]), + transaction_index: 1, + input_index: 0, + spent_txid: Txid([1; 32]), + spent_output_index: 0, + }) + } + #[test] + fn canonical_nonzero_receive_requires_durable_ack_and_survives_reopen() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "150000"); + assert_eq!( + w.transparent_pir_status()["completion"], + "checkpoint-pending" + ); + assert_eq!(w.balance().transparent_available, 0); + assert_eq!(w.balance().transparent_pending, 150000); + assert!(w + .acknowledge_transparent_pir_checkpoint(&"00".repeat(32)) + .is_err()); + let saved = w.to_snapshot().unwrap(); + let restored = WebWallet::from_snapshot(&saved).unwrap(); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "150000"); + assert_eq!( + restored.transparent_pir_status()["completion"], + "complete-for-enrolled-scope" + ); + assert_eq!(restored.history(20).len(), 1); + } + #[test] + fn receive_then_spend_offline_projects_zero_and_real_history() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![spend(), receive()]); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&c.checkpoint_id) + .unwrap(); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "0"); + assert_eq!(w.history(20).len(), 2); + assert_eq!(w.txs[&"01".repeat(32)].transparent_received, 150000); + assert_eq!(w.txs[&"02".repeat(32)].transparent_spent, 150000); + let saved = w.to_snapshot().unwrap(); + let restored = WebWallet::from_snapshot(&saved).unwrap(); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "0"); + assert_eq!(restored.history(20).len(), 2); + } + #[test] + fn transparent_pir_rejects_impossible_publisher_amount_atomically() { + let mut w = wallet(); + let mut event = receive(); + if let TransparentEvent::Receive(e) = &mut event { + e.value = u64::MAX; + } + let c = checkpoint(&mut w, vec![event]); + let before = w.to_snapshot().unwrap(); + assert!(w + .apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .is_err()); + assert_eq!(w.to_snapshot().unwrap(), before); + assert!(w.utxos.is_empty()); + } + #[test] + fn late_conflict_rolls_back_and_old_generation_cannot_apply() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + let before = w.to_snapshot().unwrap(); + let mut bad = c.clone(); + bad.records.push(TransparentPirRecord { + script: "00".into(), + event: "00".into(), + shard_id: 0, + revision: "ef".repeat(32), + }); + assert!(w + .apply_transparent_pir_checkpoint(&serde_json::to_string(&bad).unwrap()) + .is_err()); + assert_eq!(w.to_snapshot().unwrap(), before); + assert!(w.begin_transparent_pir(&"ff".repeat(32)).is_err()); + w.scanned_height = 4; + w.block_hashes.insert(4, "04".repeat(32)); + w.begin_transparent_pir(&"ab".repeat(32)).unwrap(); + assert!(w + .apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .is_err()); + assert!(w.utxos.is_empty()); + } +} + +#[cfg(test)] +mod retirement_tests { + use super::tests::*; + use super::*; + #[test] + fn transparent_pir_provisional_replacement_retires_only_projected_spend() { + let mut w = wallet(); + w.block_hashes.insert(1, "01".repeat(32)); + let c = checkpoint(&mut w, vec![receive(), spend()]); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&c.checkpoint_id) + .unwrap(); + let mut next = checkpoint(&mut w, vec![receive()]); + next.checkpoint_id = "de".repeat(32); + next.previous_checkpoint_id = Some(c.checkpoint_id); + next.rewind_to = Some(TransparentPirAnchor { + height: 1, + hash: "01".repeat(32), + }); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&next).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&next.checkpoint_id) + .unwrap(); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "150000"); + assert_eq!(w.history(20).len(), 1); + let restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "150000"); + assert_eq!(restored.history(20).len(), 1); + } + #[test] + fn transparent_pir_canonical_rewind_can_resume_partial_and_restore_received_balance() { + let mut w = wallet(); + w.block_hashes.insert(1, "01".repeat(32)); + let c = checkpoint(&mut w, vec![receive(), spend()]); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&c.checkpoint_id) + .unwrap(); + w.rewind_to_height(1).unwrap(); + let mut partial = checkpoint(&mut w, vec![]); + partial.covered_through = 1; + for c in &mut partial.coverage { + c.through = 1; + } + partial.complete = false; + partial.pending_pages = 1; + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&partial).unwrap()) + .unwrap(); + let mut restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!( + restored.transparent_pir_status()["completion"], + "incomplete" + ); + let mut complete = checkpoint(&mut restored, vec![receive()]); + complete.covered_through = 1; + for c in &mut complete.coverage { + c.through = 1; + } + complete.checkpoint_id = "de".repeat(32); + complete.previous_checkpoint_id = Some(partial.checkpoint_id); + restored + .apply_transparent_pir_checkpoint(&serde_json::to_string(&complete).unwrap()) + .unwrap(); + restored + .acknowledge_transparent_pir_checkpoint(&complete.checkpoint_id) + .unwrap(); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "150000"); + assert_eq!(restored.history(20).len(), 1); + } + #[test] + fn transparent_pir_retirement_cannot_remove_reserved_output_and_chain_rewind_invalidates() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + w.output_locks.push(StoredLock { + txid: "01".repeat(32), + pool: "transparent".into(), + index: 0, + owner: "ff".repeat(32), + expiry: 100, + }); + let mut next = checkpoint(&mut w, vec![]); + next.checkpoint_id = "de".repeat(32); + next.previous_checkpoint_id = Some(c.checkpoint_id); + next.rewind_to = Some(TransparentPirAnchor { + height: 0, + hash: Network::Regtest.genesis_hash().into(), + }); + let before = w.to_snapshot().unwrap(); + assert!(w + .apply_transparent_pir_checkpoint(&serde_json::to_string(&next).unwrap()) + .is_err()); + assert_eq!(w.to_snapshot().unwrap(), before); + assert_eq!(w.output_locks.len(), 1); + w.block_hashes.insert(1, "01".repeat(32)); + w.rewind_to_height(1).unwrap(); + assert_ne!( + w.transparent_pir_status()["completion"], + "complete-for-enrolled-scope" + ); + } +} diff --git a/crates/z-transparent-pir-v2/Cargo.toml b/crates/z-transparent-pir-v2/Cargo.toml new file mode 100644 index 0000000..6d27739 --- /dev/null +++ b/crates/z-transparent-pir-v2/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "z-transparent-pir-v2" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license = "Apache-2.0 AND MIT" +[dependencies] +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +hex = "0.4" +base64 = "0.22" +sha2 = "0.10" +bincode = "1.3" +futures = "0.3" +transparent-wallet = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-events = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-filter = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-shard = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-native = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom02 = { package = "getrandom", version = "0.2", features = ["js"] } +getrandom03 = { package = "getrandom", version = "0.3", features = ["wasm_js"] } +[dev-dependencies] +transparent-shard-server = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +tempfile = "3" +tokio = { version = "1", features = ["rt-multi-thread","sync","time"] } +axum = "0.7" +tower = { version = "0.5", features = ["util"] } diff --git a/crates/z-transparent-pir-v2/DESIGN.md b/crates/z-transparent-pir-v2/DESIGN.md new file mode 100644 index 0000000..eac3cf6 --- /dev/null +++ b/crates/z-transparent-pir-v2/DESIGN.md @@ -0,0 +1,58 @@ +# Transparent v12 acquisition contract + +The crate preserves the c9262e8 Rust walker/codec/crypto and converts only its +network suspension points to a genuinely asynchronous walker. A Pass owns that +future. `advance()` polls it and returns one owned Step; `accept(request_id, +reply)` supplies a single bounded response; no synchronous replay/cache loop. + +Public serde wire contract (snake_case `kind`, deny unknown fields): +Operation = init | map | filter {shard_id:u64} | +manifest {shard_id,revision:hex64} | +setup {shard_id,revision,table,segment:u32} | +query {shard_id,revision,table,body:base64}. +Table = {kind:"directory"} | {kind:"pages",child:u32}, child <32. +Reply = data {body:base64} | stale {map_digest_hint?:hex64} | +overloaded {retry_after?:u64} | misrouted | unavailable | refused. +Only these six public routes are admitted. Query never selects a segment. + +`Pass::new(scope: Scope, checkpoint: Option, limits: Limits)`; +`advance()->Result`; +`accept(request_id:u64, reply:Reply)->Result<()>`; +`acknowledge(checkpoint_id:&str)->Result<()>`; +`cancel()`. +Step = Exchange {request_id,operation} | NeedChain {height,hash} | +Checkpoint {checkpoint:Checkpoint} | Stopped {status:Status}. + +Scope serde fields: source_id:String (stable digest), generation:u64, +scope_id:String (wallet inventory digest), anchor:{height:u64,hash:String}, +scripts:[{script:String(hex),required_from:u64}], +accepted_headers:[{height:u64,hash:String}]. The wallet supplies its independently +accepted chain, not publisher metadata. Missing historical points yield NeedChain; +accept_chain(height,hash) only accepts independently validated wallet evidence. + +Checkpoint serde fields: source_id, generation, scope_id, +checkpoint_id:String, previous_checkpoint_id:Option, +anchor:{height,hash}, covered_through:u64, complete:bool, +pending_pages:u64, unresolved_spends:u64, +records:[{script:String(hex),event:String(hex self-contained TransparentEvent bytes), +shard_id:u64,revision:String}], +coverage:[{script:String(hex),required_from:u64,through:u64,revision:String}], +rewind_to:Option<{height,hash}>, +continuation:String(base64 versioned internal logical journal). +Records are a full source-evidence snapshot, deduplicated by upstream semantics; +the engine projects idempotently in its canonical wallet transaction. Continuation +contains durable source-evidence operations/cursors, not query secrets. No public +balance or spend authorization is exposed. Engine validates source/scope/generation +and accepted anchor before projecting and persisting continuation atomically. +Only after durable acknowledgment may a Complete status be published. Cancellation +or source/generation change retires the future and all private query contexts. + +The private source-evidence journal replays the exact upstream WalletStore +contract. It is not a second wallet balance authority: the canonical WalletDb or +WebWallet is the sole product amount/history owner. Its internal ledger is used +only by inherited completeness checks and never exported as product balance. + +Private query contexts remain opaque Rust memory between Exchange and accept. +Reopen discards contexts and regenerates fresh ciphertext from the acknowledged +logical continuation. Retries and malformed-content dummy obligations remain the +upstream walker's. Busy yields bounded delayed progress, never thread sleep. diff --git a/crates/z-transparent-pir-v2/LICENSE-APACHE b/crates/z-transparent-pir-v2/LICENSE-APACHE new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/crates/z-transparent-pir-v2/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/crates/z-transparent-pir-v2/LICENSE-FIXTURE-MIT b/crates/z-transparent-pir-v2/LICENSE-FIXTURE-MIT new file mode 100644 index 0000000..1717302 --- /dev/null +++ b/crates/z-transparent-pir-v2/LICENSE-FIXTURE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017-2021 The Electric Coin Company + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/crates/z-transparent-pir-v2/LICENSE-MIT b/crates/z-transparent-pir-v2/LICENSE-MIT new file mode 100644 index 0000000..673df49 --- /dev/null +++ b/crates/z-transparent-pir-v2/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/z-transparent-pir-v2/UPSTREAM.md b/crates/z-transparent-pir-v2/UPSTREAM.md new file mode 100644 index 0000000..db4863c --- /dev/null +++ b/crates/z-transparent-pir-v2/UPSTREAM.md @@ -0,0 +1,60 @@ +# Transparent acquisition source provenance + +The production acquisition code uses wallet-pir commit +`c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2`, the SDK's existing compatible v12 pin. +Native ReinspiRING, record codecs, filters, tags and manifests remain dependencies +at that commit. This crate implements no cryptographic arithmetic. + +Six private modules adapt `transparent/crates/transparent-wallet/src`: +`adapters.rs`, `client.rs`, `store.rs`, `memory_store.rs`, `sync.rs` and +`sync_ahead.rs`. Their original unit-test sections are omitted. The patch adds +async functions and await points, replaces synchronous transport traits with +local async traits, and changes the refusal closure to `AsyncFnMut`. It retains +the upstream directory/page walk, content-failure dummy requests, failure caps, +revision/re-cut rules and coverage checks. The initial driver permits one +outstanding exchange. The unused one-shot balance facade, default chain helper, +backoff sleep and unused store helpers are removed. Product balances remain in +the owning engine. + +An overloaded read returns incomplete immediately. The host schedules the next +bounded pass. The synchronous upstream facade instead sleeps and retries inside +its pass. No sleep, synchronous XHR, Atomics wait or synchronous replay loop is +used here. Schema/segment/size ceilings are additional host-policy refusals. + +The source-evidence store adds bounded versioned serialization. Events use the +upstream self-contained event codec. Receive/spend indexes rebuild from checked +events after restore. Query secrets and live futures are never serialized. +Rollback records an explicit independently accepted ancestor. Acknowledged +checkpoints contain a continuation and facts for the canonical wallet owner. + +`upstream-sources.json` pins the original full-file SHA256 values. +`upstream.patch` describes changes from those six files after test removal. +To reconstruct and verify the adaptation against an existing local checkout: + +```sh +python3 crates/z-transparent-pir-v2/tools/verify-upstream.py \ + /path/to/wallet-pir-at-c9262e8 +``` + +The command checks the full source hashes, strips only the original test +sections, applies the committed patch in a temporary directory and compares each +result with this crate. No checkout or dependency source is modified. Replace +this adaptation when upstream supplies an equivalent yielding or async walker +that passes the same native/WASM, differential and canonical-wallet fixtures. + +The fixture publisher and in-process server adapter in `tests/fixture.rs` adapt +`zakura/pir-transparent/src/testing/service.rs` at wallet-libraries commit +`3acb18a2f3eaff8f4739d1b26edc73095b80ff26`. The local changes allow a test to name +its independently known network/genesis, support start height zero and give cold +correctness fixtures a 180-second query deadline. This timeout is not a product +capacity or latency claim. The fixture opens no socket. Its MIT notice is in +`LICENSE-FIXTURE-MIT`. + +New Coffer SDK code is Apache-2.0. Adapted wallet-pir modules retain Roman +Akhtariev's MIT notice in each file and in `LICENSE-MIT`. The package's combined +SPDX expression is `Apache-2.0 AND MIT`. `LICENSE-APACHE` contains Apache-2.0. + +Core tests use a real native PIR server against nonzero receive/spend data, +upstream MemoryStore as a separate protocol oracle, and literal fixture events. +The engine integration adds an independent canonical full-block oracle. Fixture +agreement cannot prove that a production publisher has indexed every block. diff --git a/crates/z-transparent-pir-v2/src/adapters.rs b/crates/z-transparent-pir-v2/src/adapters.rs new file mode 100644 index 0000000..8e6a81d --- /dev/null +++ b/crates/z-transparent-pir-v2/src/adapters.rs @@ -0,0 +1,54 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! What the embedding wallet supplies besides storage. +//! +//! The library does not know the wallet's derivation rules, gap limit or +//! imported keys, and it does not decide which chain is the right one. Both +//! are handed in through these traits, so the ledger this library builds +//! follows the wallet's own policies rather than inventing parallel ones. + +use crate::store::{Anchor, ScriptEntry}; + +/// Whether a block the wallet's coverage rests on is still on its chain. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Acceptance { + Accepted, + Rejected, + /// The wallet cannot say: it has no view of this height yet. + Unknown, +} + +/// The wallet's view of the chain. Independent of the shard service by +/// construction, which is what lets a reorg be detected rather than accepted. +pub trait ChainView { + fn is_accepted(&self, height: u64, hash_display_hex: &str) -> Acceptance; + fn tip(&self) -> Option; + /// An independently accepted hash, needed for rollback inside a shard. + fn hash_at(&self, height: u64) -> Option { + self.tip().filter(|a| a.height == height).map(|a| a.hash) + } +} + +/// The wallet's script set and its rule for extending it. +pub trait ScriptProvider { + /// Every script the wallet is currently responsible for, with the height + /// its coverage must begin at. + fn scripts(&mut self) -> Vec; + /// Called with the scripts that showed activity in a sync. Returns any + /// scripts the wallet's own rules now add — a gap-limit advance — each + /// with its own required height. Those receive historical discovery + /// over their whole required range before the sync completes. + fn on_activity(&mut self, active: &[Vec]) -> Vec; +} + +/// A fixed script set that never advances. +pub struct StaticScripts(pub Vec); + +impl ScriptProvider for StaticScripts { + fn scripts(&mut self) -> Vec { + self.0.clone() + } + fn on_activity(&mut self, _: &[Vec]) -> Vec { + Vec::new() + } +} diff --git a/crates/z-transparent-pir-v2/src/client.rs b/crates/z-transparent-pir-v2/src/client.rs new file mode 100644 index 0000000..1b80dae --- /dev/null +++ b/crates/z-transparent-pir-v2/src/client.rs @@ -0,0 +1,456 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! Private retrieval of directory rows and pages from a shard set. +//! +//! One `TableClient` per geometry per table, reused across every shard naming +//! that geometry. That reuse is the point of naming geometries from a closed +//! registry rather than letting each shard choose: the expensive part of a +//! session — the parameter re-derivation and the public query setup — is done +//! once per geometry rather than once per shard. Both are the native +//! ReinspiRING two-mask profile of `transparent-native`. A sync spanning an archive +//! tier and a recent tier pays it twice, not once per shard in either. +//! +//! What *is* per segment is the published pair of masks, because it is derived +//! from that segment's own database. So a shard *revision* is "opened" by fetching the +//! setup of each of its segments, and those bytes are charged once per segment +//! per sync. A shard normally has one segment; one whose content did not fit +//! its geometry has more, and every one of them answers each query, so the +//! segment holding a script is never named in a request. +//! +//! Setup is cached against the **manifest digest**, not the shard id. A growing +//! tail is republished as a new revision with different bytes, so setup cached +//! under the id would be reused against a database it was not derived from — +//! which does not error, it decodes to plausible nonsense. +//! +//! Everything the server sends is re-derived rather than trusted. A client that +//! adopted the server's parameters would decode against whatever geometry the +//! server chose, including one that leaks the selection. +//! +//! The directory and pages use the 8-bit profile, whose one scheme is a +//! 28-bit dithered query. A 16-bit table publishes a 49-bit scheme and may +//! advertise a 44-bit dithered one beside it; the query uses the 44 bits only +//! when that scheme reproduces exactly, and otherwise the 49 bits, which are +//! still checked whole. + +use crate::transport::{BoxError, ByteCharges, Overloaded, ShardTransport, StaleRevision}; +use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, VecDeque}; +use transparent_native::{NativeScheme, NativeSecret, TableProfile}; + +#[derive(Debug, thiserror::Error)] +pub enum ClientError { + #[error("session: {0}")] + Session(String), + #[error("response: {0}")] + Response(String), + #[error("transport: {0}")] + Transport(String), + #[error("pir: {0}")] + Pir(String), + /// The service no longer serves the revision this query named. + /// + /// Kept as a type rather than folded into `Transport`: it is recoverable by + /// refreshing the map and re-deriving the shard, and a string cannot be + /// matched on. Every query goes through here, so collapsing it here would + /// lose the refusal for all of them. + #[error(transparent)] + Stale(#[from] StaleRevision), + /// The service had no cache capacity for this query. Retryable as it + /// stands, without refreshing anything. + #[error(transparent)] + Overloaded(#[from] Overloaded), + /// A reply was made under other published parameters than the setup the + /// wallet holds for that segment. The held setup is stale: the caller + /// drops it, so the next attempt fetches the published one again. + #[error("revision {revision} table {} segment {segment}: the reply was made under other parameters than the setup held", .table.name())] + SetupMismatch { + revision: String, + table: Table, + segment: u32, + }, +} + +pub use transparent_wallet::client::Table; + +/// A prepared query and the secret needed to decode its response. +pub struct PreparedQuery { + pub body: Vec, + secret: NativeSecret, +} + +/// One segment's published setup for one table: its two rounded masks, as +/// downloaded, and the epoch they imply. +struct SegmentSetup { + public: Vec, + epoch: [u8; 8], +} + +/// A client for one table kind of one geometry, across every shard. +/// +/// One pages client serves every `pages-` table: the scheme is per kind, +/// while each table's published setup, binding and stash are per table name. +pub struct TableClient { + schema: String, + /// The kind this client was derived for; see [`Table::kind`]. + kind: &'static str, + profile: TableProfile, + /// Whether queries go at 44 dithered bits, which the service advertised + /// and this client reproduced, rather than 49. + dithered: bool, + /// Per segment, because the published masks come from each segment's own + /// database. Keyed by manifest digest, table name and segment index, so a + /// superseded revision's setup is never reused for the revision that + /// replaced it, and one page shard's never for another's. + segments: HashMap<(String, String, u32), SegmentSetup>, + /// Queries a batch already sent, per revision and table, in the order the + /// walk will ask for their rows. See [`TableClient::stash`]. + stashed: HashMap<(String, String), VecDeque>, +} + +/// A query sent ahead of the walk, with its outcome. +struct Stashed { + row: usize, + query: PreparedQuery, + reply: Result, BoxError>, +} + +impl TableClient { + /// Builds a client for one table kind of `geometry`, re-deriving the + /// native parameters — bit widths, query masks and packing setup — rather + /// than adopting what the service published. + /// + /// `table` names the kind; any `Table::Pages(_)` builds the client every + /// page table of the geometry shares. `served` is the 49-bit scheme, + /// which must reproduce. `served_dithered` is the 44-bit dithered scheme, + /// if the service advertised one: queries use it only when it reproduces + /// too, and otherwise fall back to 49 bits. + + pub fn new_with_schema( + schema: &str, + table: Table, + geometry: &str, + rows: u64, + row_bytes: u32, + served: &NativeScheme, + served_dithered: Option<&NativeScheme>, + ) -> Result { + if !transparent_shard::manifest::supported_schema(schema) { + return Err(ClientError::Session("unsupported schema".into())); + } + let kind = table.kind(); + let profile = + TableProfile::new(schema, geometry, kind, rows, row_bytes).map_err(ClientError::Pir)?; + if served != &profile.scheme { + return Err(ClientError::Session(format!( + "{kind} scheme does not match the pinned geometry" + ))); + } + // Compared whole, like the 49-bit scheme. One that does not reproduce + // is not adopted; the query falls back to the 49 bits checked above. + let dithered = profile + .dithered_scheme + .as_ref() + .is_some_and(|scheme| served_dithered == Some(scheme)); + Ok(Self { + schema: schema.to_string(), + kind, + profile, + dithered, + segments: HashMap::new(), + stashed: HashMap::new(), + }) + } + + pub fn rows(&self) -> usize { + self.profile.rows + } + + /// Bits each selection coefficient is sent at: the scheme's own (28 for + /// the 8-bit tables, 49 for a 16-bit one), or 44 when a 16-bit table's + /// service advertised the dithered scheme and it reproduced. + pub fn query_bits(&self) -> usize { + match &self.profile.dithered_scheme { + Some(scheme) if self.dithered => scheme.query_bits, + _ => self.profile.scheme.query_bits, + } + } + + fn check_kind(&self, table: Table) -> Result<(), ClientError> { + if table.kind() != self.kind { + return Err(ClientError::Session(format!( + "a {} client cannot address table {}", + self.kind, + table.name() + ))); + } + Ok(()) + } + + /// Records one segment's published setup after checking it against its + /// digest. + pub fn open_segment( + &mut self, + revision: &str, + table: Table, + segment: u32, + public_params_base64: &str, + public_params_sha256: &str, + ) -> Result<(), ClientError> { + self.check_kind(table)?; + let public_params = BASE64_STANDARD + .decode(public_params_base64.as_bytes()) + .map_err(|error| ClientError::Session(error.to_string()))?; + let digest = Sha256::digest(&public_params); + if hex::encode(digest) != public_params_sha256 { + return Err(ClientError::Session(format!( + "revision {revision} published parameter digest mismatch" + ))); + } + if public_params.len() != self.profile.scheme.public_bytes { + return Err(ClientError::Session(format!( + "revision {revision} published parameter length mismatch" + ))); + } + let mut epoch = [0u8; 8]; + epoch.copy_from_slice(&digest[..8]); + self.segments.insert( + (revision.to_string(), table.name(), segment), + SegmentSetup { + public: public_params, + epoch, + }, + ); + Ok(()) + } + + /// Drops one segment's setup, so it is opened again before the next query. + pub(crate) fn forget(&mut self, revision: &str, table: Table, segment: u32) { + self.segments + .remove(&(revision.to_string(), table.name(), segment)); + } + + pub fn is_open(&self, revision: &str, table: Table, segment: u32) -> bool { + self.segments + .contains_key(&(revision.to_string(), table.name(), segment)) + } + + /// Prepares one query against a named revision and table. + /// + /// The body opens with the revision-and-table binding, which the service + /// re-derives and checks and the response repeats. It is a fixed-width + /// public value — the revision is already in the request path and in the + /// published map — so it discloses nothing while making a query answered by + /// the wrong runtime fail rather than decode. A fresh secret is sampled for + /// every query and never leaves the returned value. + pub fn prepare( + &self, + revision: &str, + table: Table, + row: usize, + ) -> Result { + self.prepare_prefix(revision, table, self.profile.rows, row) + } + + /// Prepares one query that selects among only the first `query_rows`, + /// which the revision's verified manifest fixes for every query to the + /// table (see [`PageChild::query_rows`]). Never derived from `row`: the + /// upload's length is all the service learns from it. + /// + /// [`PageChild::query_rows`]: transparent_shard::manifest::PageChild::query_rows + pub fn prepare_prefix( + &self, + revision: &str, + table: Table, + query_rows: usize, + row: usize, + ) -> Result { + self.check_kind(table)?; + if row >= query_rows || query_rows > self.profile.rows { + return Err(ClientError::Session("row outside table".into())); + } + let (secret, upload) = if self.dithered { + self.profile.prepare_dithered_prefix(query_rows, row) + } else { + self.profile.prepare_prefix(query_rows, row) + } + .map_err(ClientError::Pir)?; + let mut body = transparent_shard::manifest::query_binding_for_schema( + &self.schema, + revision, + &table.name(), + ) + .to_vec(); + body.extend(upload); + Ok(PreparedQuery { body, secret }) + } + + /// Decodes one answer: one row per segment, in segment order. + /// + /// Every segment answered the same query, so the response carries as many + /// bodies as the table has segments and the caller decides which row it + /// was looking for — by the exact script tag, never by position. + pub fn decode( + &self, + revision: &str, + table: Table, + segments: u32, + query: PreparedQuery, + response: &[u8], + ) -> Result>, ClientError> { + self.check_kind(table)?; + let name = table.name(); + let binding = + transparent_shard::manifest::query_binding_for_schema(&self.schema, revision, &name); + let each = 16 + self.profile.scheme.response_bytes; + // A fixed length per segment, and a fixed segment count from the + // verified manifest: a response of any other size is not this table's. + if response.len() != each * segments as usize { + return Err(ClientError::Response("response length mismatch".into())); + } + + let mut rows = Vec::with_capacity(segments as usize); + for segment in 0..segments { + let part = &response[segment as usize * each..(segment as usize + 1) * each]; + let setup = self + .segments + .get(&(revision.to_string(), name.clone(), segment)) + .ok_or_else(|| { + ClientError::Session(format!( + "revision {revision} table {name} segment {segment} is not open" + )) + })?; + // The prefix binds the response to the revision and table the query + // named, so an answer produced from another revision's bytes fails + // here rather than decoding to plausible rows for a range the + // wallet did not ask about. The epoch does the same for the + // segment, since each segment publishes its own masks. + if part.get(..8) != Some(binding.as_slice()) { + return Err(ClientError::Response("revision or table mismatch".into())); + } + if part.get(8..16) != Some(setup.epoch.as_slice()) { + return Err(ClientError::SetupMismatch { + revision: revision.to_string(), + table, + segment, + }); + } + let row = self + .profile + .decode(&query.secret, &setup.public, &part[16..]) + .map_err(ClientError::Response)?; + if row.len() != self.profile.row_bytes { + return Err(ClientError::Response("decoded row is too short".into())); + } + rows.push(row); + } + Ok(rows) + } + + /// Keeps a query that was sent ahead of the walk, with its outcome, for + /// the walk's next [`fetch_row`](Self::fetch_row) of that revision and + /// table. + /// + /// Stashed queries are handed out once, in order, and only for the row + /// they were prepared for: the first request that does not match drops + /// the stash and goes to the network. A stashed refusal or failure is + /// returned as if the query had just been sent, and drops the rest of the + /// stash, so whatever the walk does next — a retry, a refresh — is a + /// fresh request. + pub(crate) fn stash( + &mut self, + revision: &str, + table: Table, + row: usize, + query: PreparedQuery, + reply: Result, BoxError>, + ) { + self.stashed + .entry((revision.to_string(), table.name())) + .or_default() + .push_back(Stashed { row, query, reply }); + } + + /// Drops every stashed query. + pub(crate) fn clear_stash(&mut self) { + self.stashed.clear(); + } + + /// Fetches one row from every segment of a table, charging what it cost. + /// + /// The query selects among the first `query_rows`; see + /// [`prepare_prefix`](Self::prepare_prefix). Uses a query a batch already + /// sent for this row when one is stashed; the charge is the same either way. + #[allow(clippy::too_many_arguments)] + pub async fn fetch_row( + &mut self, + transport: &mut impl ShardTransport, + shard_id: u64, + revision: &str, + table: Table, + segments: u32, + query_rows: usize, + row: usize, + charges: &mut ByteCharges, + ) -> Result>, ClientError> { + let key = (revision.to_string(), table.name()); + if let Some(queue) = self.stashed.get_mut(&key) { + let next = queue.pop_front(); + if queue.is_empty() { + self.stashed.remove(&key); + } + match next { + Some(stashed) + if stashed.row == row + && stashed.query.body.len() + == 8 + self + .profile + .native + .request_len_bits(query_rows, self.query_bits()) => + { + let response = match stashed.reply { + Ok(response) => response, + Err(error) => { + self.stashed.remove(&key); + return Err(classify_transport(error)); + } + }; + charges.add_query( + table, + stashed.query.body.len() as u64, + response.len() as u64, + ); + return self.decode(revision, table, segments, stashed.query, &response); + } + // Out of step with the walk: nothing further is trusted to + // line up, so the rest of this table goes to the network. + _ => { + self.stashed.remove(&key); + } + } + } + let query = self.prepare_prefix(revision, table, query_rows, row)?; + let uploaded = query.body.len() as u64; + let response = transport + .query(shard_id, revision, table, &query.body) + .await + .map_err(classify_transport)?; + charges.add_query(table, uploaded, response.len() as u64); + self.decode(revision, table, segments, query, &response) + } +} + +/// Sorts a boxed transport error into the refusals a caller can act on. +/// +/// The refusals are recovered before the fallback, because both are boxed into +/// the same `BoxError` as an ordinary failure and stringifying first would +/// throw away the only thing that distinguishes them. +pub(crate) fn classify_transport(error: crate::transport::BoxError) -> ClientError { + if let Some(stale) = StaleRevision::found_in(&error) { + return ClientError::Stale(stale.clone()); + } + if let Some(overloaded) = Overloaded::found_in(&error) { + return ClientError::Overloaded(overloaded.clone()); + } + ClientError::Transport(crate::transport::describe_error(error.as_ref())) +} diff --git a/crates/z-transparent-pir-v2/src/lib.rs b/crates/z-transparent-pir-v2/src/lib.rs new file mode 100644 index 0000000..998fa44 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/lib.rs @@ -0,0 +1,18 @@ +//! Rust-owned, yielding transparent v12 acquisition. No wallet balance authority. +#![allow(async_fn_in_trait)] +mod types; +pub use types::*; +mod adapters; +mod client; +mod memory_store; +mod pass; +mod serialization; +mod store; +mod sync; +mod transport; +pub use pass::{Error, Pass}; +mod ledger { + pub use transparent_wallet::ledger::*; +} + +mod shared_store; diff --git a/crates/z-transparent-pir-v2/src/memory_store.rs b/crates/z-transparent-pir-v2/src/memory_store.rs new file mode 100644 index 0000000..a12dd40 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/memory_store.rs @@ -0,0 +1,499 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! The store, in memory: the reference semantics every other implementation +//! has to match, and what the one-shot `sync` uses. + +use crate::ledger::LedgerError; +use crate::store::{ + merge_coverage, receive_key, spend_key, Anchor, CachedRun, CoverageKind, CoverageRange, + PendingPages, ScriptEntry, SetIdentity, SetupBlob, SetupKey, ShardCommit, StoreError, + StoredEvent, WalletStore, +}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use transparent_events::{TransparentEvent, Txid}; + +/// Default bound on pending page retrievals a commit may leave. +pub const DEFAULT_PENDING_LIMIT: usize = 4_096; + +type ReceiveKey = (Txid, u32); +type SpendKey = (Txid, u32, Txid, u32); + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct State { + identity: Option, + anchor: Option, + settled_through: Option, + covered_through: Option, + scripts: BTreeMap, ScriptEntry>, + coverage: BTreeMap, Vec>, + #[serde(with = "crate::serialization::receives")] + receives: BTreeMap, + #[serde(with = "crate::serialization::spends")] + spends: BTreeMap, + /// Which spend consumes each outpoint, for the double-spend rule. + #[serde(skip)] + spent_by: BTreeMap, + pending: BTreeMap, + next_pending: u64, + setups: BTreeMap, + /// Run histories by page shard identity and script. + runs: BTreeMap<(String, Vec), CachedRun>, + filters: BTreeMap)>, + /// Content failures by shard id and script. + failures: BTreeMap<(u64, Vec), u32>, + commits: u64, +} + +/// Private source-evidence store with versioned logical continuation. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct MemoryStore { + state: State, + pending_limit: Option, +} + +impl MemoryStore { + pub fn new() -> Self { + Self::default() + } + + pub(crate) fn retain_scope(&mut self, scripts: &std::collections::BTreeSet>) { + self.state.scripts.retain(|s, _| scripts.contains(s)); + self.state.coverage.retain(|s, _| scripts.contains(s)); + self.state + .receives + .retain(|_, e| scripts.contains(&e.script)); + self.state.spends.retain(|_, e| scripts.contains(&e.script)); + self.state + .pending + .retain(|_, p| scripts.contains(&p.script)); + self.state.runs.retain(|(_, s), _| scripts.contains(s)); + self.rebuild_indexes(); + } + + pub(crate) fn rebuild_indexes(&mut self) { + self.state.spent_by.clear(); + for stored in self.state.spends.values() { + if let TransparentEvent::Spend(event) = stored.event { + self.state.spent_by.insert( + (event.spent_txid, event.spent_output_index), + spend_key(&event), + ); + } + } + } + + fn height_of(event: &TransparentEvent) -> u64 { + match event { + TransparentEvent::Receive(receive) => u64::from(receive.height), + TransparentEvent::Spend(spend) => u64::from(spend.height), + } + } +} + +impl WalletStore for MemoryStore { + fn set_identity(&self) -> Result, StoreError> { + Ok(self.state.identity.clone()) + } + + fn bind_set(&mut self, identity: &SetIdentity) -> Result<(), StoreError> { + match &self.state.identity { + None => { + self.state.identity = Some(identity.clone()); + Ok(()) + } + Some(stored) if stored.continues(identity) => { + // A geometry the store had not seen may appear; record it. + self.state.identity = Some(identity.clone()); + Ok(()) + } + Some(stored) => Err(StoreError::SetMismatch { + stored: stored.digest(), + offered: identity.digest(), + }), + } + } + + fn anchor(&self) -> Result, StoreError> { + Ok(self.state.anchor.clone()) + } + + fn scripts(&self) -> Result, StoreError> { + Ok(self.state.scripts.values().cloned().collect()) + } + + fn add_scripts(&mut self, entries: &[ScriptEntry]) -> Result { + let mut added = 0; + for entry in entries { + match self.state.scripts.get_mut(&entry.script) { + Some(existing) => { + // Coverage once required stays required. + existing.required_from = existing.required_from.min(entry.required_from); + } + None => { + self.state + .scripts + .insert(entry.script.clone(), entry.clone()); + added += 1; + } + } + } + Ok(added) + } + + fn coverage(&self, script: &[u8]) -> Result, StoreError> { + Ok(self.state.coverage.get(script).cloned().unwrap_or_default()) + } + + fn provisional(&self) -> Result, StoreError> { + Ok(self + .state + .coverage + .values() + .flatten() + .filter(|range| range.kind == CoverageKind::Provisional) + .cloned() + .collect()) + } + + fn events(&self) -> Result, StoreError> { + let mut events: Vec = self + .state + .receives + .values() + .chain(self.state.spends.values()) + .cloned() + .collect(); + events.sort_by_key(|entry| entry.event.sort_key()); + Ok(events) + } + + fn commit_shard(&mut self, commit: ShardCommit) -> Result { + transparent_events::check_transaction_consistency( + self.state + .receives + .values() + .chain(self.state.spends.values()) + .chain(commit.events.iter()) + .map(|stored| &stored.event), + ) + .map_err(|e| StoreError::Corrupt(e.to_string()))?; + // Validate everything against the current state first; only then + // write, so a refused commit leaves the store exactly as it was. + let mut new_receives: Vec<(ReceiveKey, StoredEvent)> = Vec::new(); + let mut new_spends: Vec<(SpendKey, StoredEvent)> = Vec::new(); + for stored in &commit.events { + match &stored.event { + TransparentEvent::Receive(receive) => { + let key = receive_key(receive); + if let Some(existing) = self.state.receives.get(&key) { + if existing.event != stored.event || existing.script != stored.script { + return Err(LedgerError::ConflictingReceive( + receive.txid.to_display_hex(), + receive.output_index, + ) + .into()); + } + continue; + } + new_receives.push((key, stored.clone())); + } + TransparentEvent::Spend(spend) => { + let key = spend_key(spend); + let outpoint = (spend.spent_txid, spend.spent_output_index); + if let Some(existing) = self.state.spends.get(&key) { + if existing.event != stored.event || existing.script != stored.script { + return Err(LedgerError::DoubleSpend( + spend.spent_txid.to_display_hex(), + spend.spent_output_index, + ) + .into()); + } + continue; + } + if let Some(other) = self.state.spent_by.get(&outpoint) { + if *other != key { + return Err(LedgerError::DoubleSpend( + spend.spent_txid.to_display_hex(), + spend.spent_output_index, + ) + .into()); + } + } + new_spends.push((key, stored.clone())); + } + } + } + let limit = self.pending_limit(); + let pending_after = self.state.pending.len() + + commit + .pending_upsert + .iter() + .filter(|p| p.id.is_none()) + .count() + - commit.pending_complete.len().min(self.state.pending.len()); + if pending_after > limit { + return Err(StoreError::PendingLimit { limit }); + } + for run in &commit.cached_runs { + if self + .state + .runs + .get(&(run.child_digest.clone(), run.script.clone())) + .is_some_and(|held| held != run) + { + return Err(StoreError::Corrupt(format!( + "page shard {} holds two histories for one script", + run.child_digest + ))); + } + } + + for (key, event) in new_receives { + self.state.receives.insert(key, event); + } + for (key, event) in new_spends { + let outpoint = (key.2, key.3); + self.state.spent_by.insert(outpoint, key); + self.state.spends.insert(key, event); + } + let kind = if commit.sealed { + CoverageKind::Settled + } else { + CoverageKind::Provisional + }; + for script in &commit.covered_scripts { + let ranges = self.state.coverage.entry(script.clone()).or_default(); + let range = CoverageRange { + source_anchor: commit.source_anchor.clone(), + script: script.clone(), + start_height: commit.start_height, + end_height: commit.end_height, + kind, + shard_id: commit.shard_id, + revision_digest: commit.revision_digest.clone(), + terminal_block_hash: commit.terminal_block_hash.clone(), + }; + // A range strictly inside the one starting nearest at or below it + // is already covered: nothing changes. Otherwise it replaces every + // range of this script it contains, whatever shard they came + // from: a repeat, a longer revision at the same start, and the + // narrower shards a re-cut merged into this one. The SQLite store + // keeps the same rule. + let holder = ranges + .iter() + .filter(|old| old.start_height <= range.start_height) + .max_by_key(|old| (old.start_height, old.end_height)); + if holder.is_some_and(|old| { + old.end_height >= range.end_height + && (old.start_height, old.end_height) != (range.start_height, range.end_height) + }) { + continue; + } + ranges.retain(|old| { + !(old.start_height >= range.start_height && old.end_height <= range.end_height) + }); + if !ranges.contains(&range) { + ranges.push(range); + } + let merged = merge_coverage(std::mem::take(ranges)); + *ranges = merged; + } + for id in &commit.pending_complete { + self.state.pending.remove(id); + } + let finishes_shard = commit.finishes_shard(); + for run in commit.cached_runs { + self.state + .runs + .insert((run.child_digest.clone(), run.script.clone()), run); + } + if finishes_shard { + self.state.runs.retain(|(_, script), run| { + run.shard_id != commit.shard_id || !commit.covered_scripts.contains(script) + }); + } + for script in commit.failed_entries { + *self + .state + .failures + .entry((commit.shard_id, script)) + .or_default() += 1; + } + for mut pending in commit.pending_upsert { + let id = match pending.id { + Some(id) => id, + None => { + self.state.next_pending += 1; + self.state.next_pending + } + }; + pending.id = Some(id); + self.state.pending.insert(id, pending); + } + self.state.commits += 1; + Ok(self.state.commits) + } + + fn commit_anchor( + &mut self, + anchor: &Anchor, + settled_through: u64, + covered_through: u64, + ) -> Result { + self.state.anchor = Some(anchor.clone()); + self.state.settled_through = Some(settled_through); + self.state.covered_through = Some(covered_through); + self.state.commits += 1; + Ok(self.state.commits) + } + + fn rollback_above(&mut self, accepted: &Anchor, _reason: &str) -> Result { + let height = accepted.height; + self.state + .receives + .retain(|_, stored| Self::height_of(&stored.event) <= height); + let mut removed_spends: Vec = Vec::new(); + self.state.spends.retain(|key, stored| { + let keep = Self::height_of(&stored.event) <= height; + if !keep { + removed_spends.push(*key); + } + keep + }); + for key in removed_spends { + self.state.spent_by.remove(&(key.2, key.3)); + } + for ranges in self.state.coverage.values_mut() { + ranges.retain_mut(|range| { + if range.start_height > height { + return false; + } + if range.end_height > height { + // A range straddling the rollback keeps its covered part; + // the identity it carried no longer describes the whole + // range, but it still names the shard the part came from. + range.end_height = height; + range.terminal_block_hash = accepted.hash.clone(); + } + true + }); + } + // Cached runs survive: each is keyed by its page shard's chained + // identity, which commits to that page shard's terminal block and to + // every earlier one, so a walk reaches it only through a manifest on + // the accepted chain. A tail replaced by a later revision keeps the + // runs its sealed page shards share, which is what lets the catch-up + // stop at them. + // All pending retrievals are inexpensive to re-plan and may contain old-branch data. + self.state.pending.clear(); + if let Some(anchor) = &mut self.state.anchor { + if anchor.height > height { + anchor.height = height; + anchor.hash = accepted.hash.clone(); + } + } + if let Some(settled) = &mut self.state.settled_through { + *settled = (*settled).min(height); + } + if let Some(covered) = &mut self.state.covered_through { + *covered = (*covered).min(height); + } + self.state.commits += 1; + Ok(self.state.commits) + } + + fn promote_provisional( + &mut self, + shard_id: u64, + revision_digest: &str, + ) -> Result<(), StoreError> { + for ranges in self.state.coverage.values_mut() { + for range in ranges.iter_mut() { + if range.shard_id == shard_id + && range.revision_digest == revision_digest + && range.kind == CoverageKind::Provisional + { + range.kind = CoverageKind::Settled; + } + } + let merged = merge_coverage(std::mem::take(ranges)); + *ranges = merged; + } + Ok(()) + } + + fn pending(&self) -> Result, StoreError> { + Ok(self.state.pending.values().cloned().collect()) + } + + fn cached_run( + &self, + child_digest: &str, + script: &[u8], + ) -> Result, StoreError> { + Ok(self + .state + .runs + .get(&(child_digest.to_string(), script.to_vec())) + .cloned()) + } + + fn pending_limit(&self) -> usize { + self.pending_limit.unwrap_or(DEFAULT_PENDING_LIMIT) + } + + fn setup(&self, key: &SetupKey) -> Result, StoreError> { + Ok(self.state.setups.get(key).cloned()) + } + + fn put_setup(&mut self, key: &SetupKey, blob: &SetupBlob) -> Result<(), StoreError> { + self.state.setups.insert(key.clone(), blob.clone()); + Ok(()) + } + + fn drop_setup(&mut self, key: &SetupKey) -> Result<(), StoreError> { + self.state.setups.remove(key); + Ok(()) + } + + fn filter( + &self, + revision_digest: &str, + filter_hash: &str, + ) -> Result>, StoreError> { + Ok(self + .state + .filters + .get(revision_digest) + .filter(|(hash, _, _)| hash == filter_hash) + .map(|(_, _, bytes)| bytes.clone())) + } + + fn put_filter( + &mut self, + revision_digest: &str, + filter_hash: &str, + sealed: bool, + bytes: &[u8], + ) -> Result<(), StoreError> { + self.state.filters.insert( + revision_digest.to_string(), + (filter_hash.to_string(), sealed, bytes.to_vec()), + ); + Ok(()) + } + + fn last_commit(&self) -> Result { + Ok(self.state.commits) + } + + fn entry_failures(&self, shard_id: u64, script: &[u8]) -> Result { + Ok(self + .state + .failures + .get(&(shard_id, script.to_vec())) + .copied() + .unwrap_or(0)) + } +} diff --git a/crates/z-transparent-pir-v2/src/pass.rs b/crates/z-transparent-pir-v2/src/pass.rs new file mode 100644 index 0000000..a84d509 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/pass.rs @@ -0,0 +1,741 @@ +use crate::{ + adapters::{Acceptance, ChainView, StaticScripts}, + client::Table as NativeTable, + memory_store::MemoryStore, + shared_store::SharedStore, + store::{ScriptEntry, ScriptOrigin, WalletStore}, + sync::{self, Completion, GeometryParams, ServiceGeometry, WorkLimits}, + transport::{BoxError, FilterSource, Overloaded, ShardTransport, StaleRevision}, + types::*, +}; +use base64::{engine::general_purpose::STANDARD, Engine}; +use bincode::Options; +use futures::{ + future::{poll_fn, LocalBoxFuture}, + task::noop_waker_ref, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + cell::RefCell, + collections::BTreeMap, + rc::Rc, + task::{Context, Poll}, + time::Duration, +}; +#[derive(Serialize, Deserialize)] +struct Continuation { + version: u32, + source_id: String, + state: MemoryStore, +} +fn reason_code(r: sync::IncompleteReason) -> &'static str { + use sync::IncompleteReason::*; + match r { + QueryBudget => "query_budget", + ByteBudget => "byte_budget", + PendingLimit => "pending_limit", + Overloaded { .. } => "overloaded", + ChainUnknown { .. } => "chain_unknown", + DiscoveryUnbounded => "scope_growing", + ContentFailed => "content_failed", + PublicationBehind { .. } => "publication_lag", + UnresolvedSpends => "unresolved_spends", + } +} + +#[derive(Debug, thiserror::Error)] +#[error("transparent recovery: {0}")] +pub struct Error(pub(crate) &'static str); +type Result = std::result::Result; +#[derive(Clone)] +enum Pending { + Exchange { id: u64, operation: Operation }, + Chain { height: u64, hash: String }, +} +#[derive(Default)] +struct Io { + next: u64, + pending: Option, + reply: Option, + chain_reply: Option, + exchanges: u64, + download: u64, + upload: u64, + stop: Option<&'static str>, +} +#[derive(Clone)] +struct Driver { + io: Rc>, + limits: Limits, + chain: Rc>>, +} +impl Driver { + async fn send(&self, operation: Operation) -> std::result::Result, BoxError> { + let upload = if let Operation::Query { body, .. } = &operation { + body.len() as u64 + } else { + 0 + }; + { + let mut io = self.io.borrow_mut(); + if io.exchanges >= self.limits.max_exchanges + || io.upload.saturating_add(upload) > self.limits.max_upload_bytes + { + io.stop = Some("budget"); + return Err(Box::new(Error("budget"))); + } + io.exchanges += 1; + io.upload += upload; + io.next += 1; + let id = io.next; + io.pending = Some(Pending::Exchange { + id, + operation: operation.clone(), + }); + } + let reply = poll_fn(|_| { + let mut io = self.io.borrow_mut(); + match io.reply.take() { + Some(r) => { + io.pending = None; + Poll::Ready(r) + } + None => Poll::Pending, + } + }) + .await; + match reply { + Reply::Data { body } => { + let max = match operation { + Operation::Filter { .. } => 8 * 1024 * 1024, + Operation::Setup { .. } => 128 * 1024, + Operation::Query { .. } => 512 * 1024, + _ => 1024 * 1024, + }; + if body.len() > max * 4 / 3 + 8 { + return Err(Box::new(Error("response_limit"))); + } + let bytes = STANDARD + .decode(body) + .map_err(|_| Error("response_encoding"))?; + if bytes.len() > max { + return Err(Box::new(Error("response_limit"))); + } + Ok(bytes) + } + Reply::Stale { map_digest_hint } => { + let (shard_id, revision) = match operation { + Operation::Manifest { shard_id, revision } + | Operation::Setup { + shard_id, revision, .. + } + | Operation::Query { + shard_id, revision, .. + } => (shard_id, revision), + _ => (0, String::new()), + }; + Err(Box::new(StaleRevision { + shard_id, + revision, + map_sha256: map_digest_hint, + })) + } + Reply::Overloaded { retry_after } => Err(Box::new(Overloaded { + retry_after: retry_after.map(|s| Duration::from_secs(s.clamp(1, 60))), + })), + Reply::Misrouted => Err(Box::new(Error("misrouted"))), + Reply::Unavailable => Err(Box::new(Error("unavailable"))), + Reply::Refused => Err(Box::new(Error("refused"))), + } + } + async fn point(&self, height: u64, expected: &str) -> Result<()> { + if self.chain.borrow().contains_key(&height) { + return Ok(()); + } + self.io.borrow_mut().pending = Some(Pending::Chain { + height, + hash: expected.to_string(), + }); + let point = poll_fn(|_| { + let mut io = self.io.borrow_mut(); + match io.chain_reply.take() { + Some(a) => { + io.pending = None; + Poll::Ready(a) + } + None => Poll::Pending, + } + }) + .await; + if point.height != height { + return Err(Error("chain_point")); + } + self.chain.borrow_mut().insert(height, point.hash); + Ok(()) + } +} +fn native_table(t: NativeTable) -> Table { + match t { + NativeTable::Directory => Table::Directory, + NativeTable::Pages(child) => Table::Pages { child }, + } +} +impl FilterSource for Driver { + async fn shard_map(&mut self) -> std::result::Result<(Vec, u64), BoxError> { + let b = self.send(Operation::Map).await?; + let n = b.len() as u64; + Ok((b, n)) + } + async fn filter(&mut self, shard_id: u64) -> std::result::Result<(Vec, u64), BoxError> { + let b = self.send(Operation::Filter { shard_id }).await?; + let n = b.len() as u64; + Ok((b, n)) + } +} +impl ShardTransport for Driver { + async fn init(&mut self) -> std::result::Result<(Vec, u64), BoxError> { + let b = self.send(Operation::Init).await?; + let n = b.len() as u64; + Ok((b, n)) + } + async fn manifest( + &mut self, + shard_id: u64, + revision: &str, + ) -> std::result::Result<(Vec, u64), BoxError> { + let b = self + .send(Operation::Manifest { + shard_id, + revision: revision.to_string(), + }) + .await?; + let n = b.len() as u64; + Ok((b, n)) + } + async fn setup( + &mut self, + shard_id: u64, + revision: &str, + table: NativeTable, + segment: u32, + ) -> std::result::Result<(Vec, u64), BoxError> { + let b = self + .send(Operation::Setup { + shard_id, + revision: revision.to_string(), + table: native_table(table), + segment, + }) + .await?; + let n = b.len() as u64; + Ok((b, n)) + } + async fn query( + &mut self, + shard_id: u64, + revision: &str, + table: NativeTable, + body: &[u8], + ) -> std::result::Result, BoxError> { + self.send(Operation::Query { + shard_id, + revision: revision.to_string(), + table: native_table(table), + body: STANDARD.encode(body), + }) + .await + } +} +struct Chain(Rc>>); +impl ChainView for Chain { + fn is_accepted(&self, h: u64, s: &str) -> Acceptance { + match self.0.borrow().get(&h) { + Some(held) if held == s => Acceptance::Accepted, + Some(_) => Acceptance::Rejected, + None => Acceptance::Unknown, + } + } + fn hash_at(&self, h: u64) -> Option { + self.0.borrow().get(&h).cloned() + } + fn tip(&self) -> Option { + self.0 + .borrow() + .iter() + .next_back() + .map(|(h, s)| crate::store::Anchor { + height: *h, + hash: s.clone(), + }) + } +} + +pub struct Pass { + scope: Scope, + store: SharedStore, + driver: Driver, + future: Option>>, + previous: Option, + baseline: Vec, + held: Option, + finished: Option, + cancelled: bool, + final_ack: bool, +} +fn encoded(store: &MemoryStore) -> Result> { + bincode::DefaultOptions::new() + .with_fixint_encoding() + .serialize(store) + .map_err(|_| Error("checkpoint_encode")) +} +fn hash(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +fn checkpoint_id( + source: &str, + scope: &str, + generation: u64, + anchor: &Anchor, + complete: bool, + bytes: &[u8], +) -> String { + let mut digest = Sha256::new(); + digest.update(source.as_bytes()); + digest.update(scope.as_bytes()); + digest.update(generation.to_le_bytes()); + digest.update(anchor.height.to_le_bytes()); + digest.update(anchor.hash.as_bytes()); + digest.update([u8::from(complete)]); + digest.update(bytes); + hex::encode(digest.finalize()) +} +impl Pass { + pub fn new(scope: Scope, checkpoint: Option, limits: Limits) -> Result { + if limits.max_exchanges == 0 + || limits.max_exchanges > 4096 + || limits.max_scripts > 4096 + || !hash(&scope.source_id) + || !hash(&scope.scope_id) + || !hash(&scope.anchor.hash) + || scope.accepted_headers.len() > 4096 + || scope.scripts.len() > limits.max_scripts + || scope.scripts.is_empty() + { + return Err(Error("scope")); + } + let mut seen = std::collections::BTreeSet::new(); + if scope.scripts.iter().any(|s| !seen.insert(&s.script)) { + return Err(Error("duplicate_script")); + } + let mut memory = if let Some(cp) = &checkpoint { + if cp.source_id != scope.source_id { + return Err(Error("source_changed")); + } + if cp.continuation.len() > 96 * 1024 * 1024 { + return Err(Error("checkpoint_limit")); + } + let bytes = STANDARD + .decode(&cp.continuation) + .map_err(|_| Error("checkpoint_encoding"))?; + let restored: Continuation = bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(64 * 1024 * 1024) + .reject_trailing_bytes() + .deserialize(&bytes) + .map_err(|_| Error("checkpoint_decode"))?; + if restored.version != 1 || restored.source_id != scope.source_id { + return Err(Error("checkpoint_source")); + } + let mut state = restored.state; + state.rebuild_indexes(); + state + } else { + MemoryStore::new() + }; + if let Some(cp) = &checkpoint { + let original = encoded(&memory)?; + if checkpoint_id( + &cp.source_id, + &cp.scope_id, + cp.generation, + &cp.anchor, + cp.complete, + &original, + ) != cp.checkpoint_id + { + return Err(Error("checkpoint_id")); + } + } + let current_scripts = scope + .scripts + .iter() + .map(|s| hex::decode(&s.script).map_err(|_| Error("script"))) + .collect::>>()?; + memory.retain_scope(¤t_scripts); + let baseline = encoded(&memory)?; + let store = SharedStore { + inner: Rc::new(RefCell::new(memory)), + rewind: Rc::new(RefCell::new(None)), + }; + let mut headers = BTreeMap::new(); + for point in &scope.accepted_headers { + if !hash(&point.hash) { + return Err(Error("chain")); + } + if let Some(old) = headers.insert(point.height, point.hash.clone()) { + if old != point.hash { + return Err(Error("chain")); + } + } + } + if headers.get(&scope.anchor.height) != Some(&scope.anchor.hash) { + return Err(Error("anchor_not_accepted")); + } + let driver = Driver { + io: Rc::new(RefCell::new(Io::default())), + limits: limits.clone(), + chain: Rc::new(RefCell::new(headers)), + }; + let mut filters = driver.clone(); + let mut transport = driver.clone(); + let owned = driver.clone(); + let chain = Chain(driver.chain.clone()); + let mut state = store.clone(); + let request = scope.clone(); + let future = Box::pin(async move { + let (init, _) = transport.init().await.map_err(|_| Error("init"))?; + let parsed = + transparent_wallet::parse_init(&init).map_err(|_| Error("init_invalid"))?; + let geometry = ServiceGeometry { + schema: parsed.schema, + geometries: parsed + .geometries + .into_iter() + .map(|g| GeometryParams { + name: g.name, + directory_rows: g.directory_rows, + directory_row_bytes: g.directory_row_bytes, + directory_scheme: g.directory_scheme, + directory_scheme_dq44: g.directory_scheme_dq44, + directory_setup_seed: g.directory_setup_seed, + page_rows: g.page_rows, + page_row_bytes: g.page_row_bytes, + pages_scheme: g.pages_scheme, + pages_scheme_dq44: g.pages_scheme_dq44, + pages_setup_seed: g.pages_setup_seed, + }) + .collect(), + }; + let (raw, cost) = filters.shard_map().await.map_err(|_| Error("map"))?; + let map: transparent_filter::ShardMap = + serde_json::from_slice(&raw).map_err(|_| Error("map_invalid"))?; + map.check_shape().map_err(|_| Error("map_invalid"))?; + let genesis = match map.network.as_str() { + "main" | "mainnet" => transparent_filter::MAINNET_GENESIS_DISPLAY, + "test" | "testnet" => { + "05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38" + } + "regtest" => "029f11d80ef9765602235e1bc9727e3eb6ba20839319f761fee920d63401e327", + _ => { + return Ok(Status::Incomplete { + reason: "publication_profile".into(), + }) + } + }; + if map.genesis_hash != genesis + || map.profile != transparent_filter::RANGE_PROFILE_V2.name + { + return Ok(Status::Incomplete { + reason: "publication_profile".into(), + }); + } + owned.point(0, genesis).await?; + if !matches!(chain.is_accepted(0, genesis), Acceptance::Accepted) { + return Ok(Status::Incomplete { + reason: "chain_identity".into(), + }); + } + if map.shards.len() > 1024 + || request + .scripts + .iter() + .any(|s| s.required_from < map.start_height) + { + return Ok(Status::Incomplete { + reason: "publication_incomplete".into(), + }); + } + for e in &map.shards { + if e.end_height <= request.anchor.height { + owned.point(e.end_height, &e.terminal_block_hash).await?; + } + if e.start_height > 0 && e.start_height - 1 <= request.anchor.height { + owned + .point(e.start_height - 1, &e.parent_block_hash) + .await?; + } + } + for e in state.scripts().map_err(|_| Error("store"))? { + for range in state.coverage(&e.script).map_err(|_| Error("store"))? { + owned + .point(range.end_height, &range.terminal_block_hash) + .await?; + } + } + let scripts = request + .scripts + .iter() + .map(|s| { + Ok(ScriptEntry { + script: hex::decode(&s.script).map_err(|_| Error("script"))?, + origin: ScriptOrigin::Derived, + required_from: s.required_from, + }) + }) + .collect::>>()?; + if scripts.iter().any(|s| { + s.script.is_empty() + || s.script.len() > 40 + || s.required_from > request.anchor.height + }) { + return Err(Error("script")); + } + let mut scripts = StaticScripts(scripts); + let target = crate::store::Anchor { + height: request.anchor.height, + hash: request.anchor.hash, + }; + match sync::sync_into( + &mut state, + &map, + cost, + &geometry, + &chain, + &mut scripts, + &mut filters, + &mut transport, + &WorkLimits { + max_queries: Some(limits.max_queries), + max_private_bytes: Some(limits.max_private_bytes), + }, + &target, + ) + .await + { + Ok(report) => Ok(match report.completion { + Completion::Complete => Status::Complete, + Completion::Incomplete { reason, .. } => Status::Incomplete { + reason: reason_code(reason).into(), + }, + }), + Err(_) => Ok(Status::Incomplete { + reason: "recovery_failed".into(), + }), + } + }); + Ok(Self { + scope, + store, + driver, + future: Some(future), + previous: checkpoint.map(|c| c.checkpoint_id), + baseline, + held: None, + finished: None, + cancelled: false, + final_ack: false, + }) + } + pub fn advance(&mut self) -> Result { + if self.cancelled { + return Ok(Step::Stopped { + status: Status::Cancelled, + }); + } + if let Some(cp) = &self.held { + return Ok(Step::Checkpoint { + checkpoint: cp.clone(), + }); + } + if self.finished.is_none() { + if let Some(future) = self.future.as_mut() { + let mut cx = Context::from_waker(noop_waker_ref()); + if let Poll::Ready(result) = future.as_mut().poll(&mut cx) { + self.finished = Some(if let Some(reason) = self.driver.io.borrow().stop { + Status::Incomplete { + reason: reason.into(), + } + } else { + result.unwrap_or_else(|_| Status::Incomplete { + reason: "unavailable".into(), + }) + }); + self.future = None; + } + } + } + let bytes = encoded(&self.store.inner.borrow())?; + if bytes != self.baseline || self.finished.is_some() { + // A final status checkpoint is emitted once, then removed by acknowledge. + if self.finished.is_some() + && self.future.is_none() + && self.baseline == bytes + && self.final_ack + { + return Ok(Step::Stopped { + status: self.finished.clone().unwrap(), + }); + } + let cp = self.checkpoint(&bytes)?; + self.held = Some(cp.clone()); + return Ok(Step::Checkpoint { checkpoint: cp }); + } + match self.driver.io.borrow().pending.clone() { + Some(Pending::Exchange { id, operation }) => Ok(Step::Exchange { + request_id: id, + operation, + }), + Some(Pending::Chain { height, hash }) => Ok(Step::NeedChain { height, hash }), + None => Err(Error("no_progress")), + } + } + fn checkpoint(&self, bytes: &[u8]) -> Result { + let state = self.store.inner.borrow(); + let complete = self.finished == Some(Status::Complete); + let records = state + .events() + .map_err(|_| Error("store"))? + .into_iter() + .map(|e| Record { + script: hex::encode(e.script), + event: hex::encode(e.event.to_bytes()), + shard_id: e.shard_id, + revision: e.revision_digest, + }) + .collect(); + if state.events().map_err(|_| Error("store"))?.len() > 65536 { + return Err(Error("event_limit")); + } + let mut coverage = Vec::new(); + let mut through = self.scope.anchor.height; + for script in state.scripts().map_err(|_| Error("store"))? { + let ranges = state.coverage(&script.script).map_err(|_| Error("store"))?; + let mut end = script.required_from.saturating_sub(1); + let mut revision = String::new(); + for r in ranges { + if r.start_height <= end.saturating_add(1) && r.end_height > end { + end = r.end_height; + revision = r.revision_digest + } + } + through = through.min(end); + if end >= script.required_from && !revision.is_empty() { + coverage.push(Coverage { + script: hex::encode(script.script), + required_from: script.required_from, + through: end, + revision, + }); + } + } + let durable = bincode::DefaultOptions::new() + .with_fixint_encoding() + .serialize(&Continuation { + version: 1, + source_id: self.scope.source_id.clone(), + state: state.clone(), + }) + .map_err(|_| Error("checkpoint_encode"))?; + if durable.len() > 64 * 1024 * 1024 { + return Err(Error("checkpoint_limit")); + } + let continuation = STANDARD.encode(durable); + let id = checkpoint_id( + &self.scope.source_id, + &self.scope.scope_id, + self.scope.generation, + &self.scope.anchor, + complete, + bytes, + ); + Ok(Checkpoint { + source_id: self.scope.source_id.clone(), + generation: self.scope.generation, + scope_id: self.scope.scope_id.clone(), + checkpoint_id: id, + previous_checkpoint_id: self.previous.clone(), + anchor: self.scope.anchor.clone(), + covered_through: through, + complete, + pending_pages: state.pending().map_err(|_| Error("store"))?.len() as u64, + unresolved_spends: state + .ledger() + .map_err(|_| Error("store"))? + .unresolved() + .len() as u64, + records, + coverage, + rewind_to: self.store.rewind.borrow().clone().map(|a| Anchor { + height: a.height, + hash: a.hash, + }), + continuation, + }) + } + pub fn accept(&mut self, request_id: u64, reply: Reply) -> Result<()> { + if self.held.is_some() || self.cancelled { + return Err(Error("not_waiting")); + } + let mut io = self.driver.io.borrow_mut(); + if !matches!(io.pending,Some(Pending::Exchange{id,..})if id==request_id) + || io.reply.is_some() + { + return Err(Error("request_id")); + } + let bytes = serde_json::to_vec(&reply) + .map_err(|_| Error("reply"))? + .len() as u64; + io.download = io.download.saturating_add(bytes); + if io.download > self.driver.limits.max_download_bytes { + io.stop = Some("budget"); + io.reply = Some(Reply::Unavailable) + } else { + io.reply = Some(reply) + } + Ok(()) + } + pub fn accept_chain(&mut self, height: u64, hash: String) -> Result<()> { + if !self.held.is_none() || !crate::pass::hash(&hash) { + return Err(Error("chain_point")); + } + let mut io = self.driver.io.borrow_mut(); + if !matches!(io.pending,Some(Pending::Chain{height:h,..})if h==height) + || io.chain_reply.is_some() + { + return Err(Error("chain_point")); + } + io.chain_reply = Some(Anchor { height, hash }); + Ok(()) + } + pub fn acknowledge(&mut self, id: &str) -> Result<()> { + let cp = self.held.as_ref().ok_or(Error("checkpoint_missing"))?; + if cp.checkpoint_id != id { + return Err(Error("checkpoint_id")); + } + self.baseline = encoded(&self.store.inner.borrow())?; + self.previous = Some(id.into()); + self.final_ack = self.finished.is_some(); + self.store.rewind.borrow_mut().take(); + self.held = None; + Ok(()) + } + pub fn cancel(&mut self) { + self.cancelled = true; + self.future = None; + self.held = None; + self.driver.io.borrow_mut().pending = None; + } +} diff --git a/crates/z-transparent-pir-v2/src/serialization.rs b/crates/z-transparent-pir-v2/src/serialization.rs new file mode 100644 index 0000000..320a7f5 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/serialization.rs @@ -0,0 +1,89 @@ +use crate::store::{receive_key, spend_key, StoredEvent}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::collections::BTreeMap; +use transparent_events::TransparentEvent; +pub(crate) mod event { + use super::*; + pub fn serialize(v: &TransparentEvent, s: S) -> Result { + v.to_bytes().serialize(s) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + let b = Vec::::deserialize(d)?; + TransparentEvent::from_bytes(&b).map_err(serde::de::Error::custom) + } +} +pub(crate) mod events { + use super::*; + pub fn serialize(v: &[TransparentEvent], s: S) -> Result { + v.iter() + .map(|e| e.to_bytes()) + .collect::>() + .serialize(s) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + Vec::>::deserialize(d)? + .iter() + .map(|b| TransparentEvent::from_bytes(b).map_err(serde::de::Error::custom)) + .collect() + } +} +pub(crate) mod table { + use super::*; + use crate::client::Table; + pub fn serialize(v: &Table, s: S) -> Result { + match v { + Table::Directory => None, + Table::Pages(i) => Some(*i), + } + .serialize(s) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + Ok(match Option::::deserialize(d)? { + None => Table::Directory, + Some(i) if i < 32 => Table::Pages(i), + _ => return Err(serde::de::Error::custom("invalid table")), + }) + } +} +pub(crate) mod receives { + use super::*; + use transparent_events::Txid; + pub fn serialize( + v: &BTreeMap<(Txid, u32), StoredEvent>, + s: S, + ) -> Result { + v.values().collect::>().serialize(s) + } + pub fn deserialize<'de, D: Deserializer<'de>>( + d: D, + ) -> Result, D::Error> { + Vec::::deserialize(d)? + .into_iter() + .map(|e| match e.event { + TransparentEvent::Receive(r) => Ok((receive_key(&r), e)), + _ => Err(serde::de::Error::custom("invalid receive")), + }) + .collect() + } +} +pub(crate) mod spends { + use super::*; + use transparent_events::Txid; + pub fn serialize( + v: &BTreeMap<(Txid, u32, Txid, u32), StoredEvent>, + s: S, + ) -> Result { + v.values().collect::>().serialize(s) + } + pub fn deserialize<'de, D: Deserializer<'de>>( + d: D, + ) -> Result, D::Error> { + Vec::::deserialize(d)? + .into_iter() + .map(|e| match e.event { + TransparentEvent::Spend(r) => Ok((spend_key(&r), e)), + _ => Err(serde::de::Error::custom("invalid spend")), + }) + .collect() + } +} diff --git a/crates/z-transparent-pir-v2/src/shared_store.rs b/crates/z-transparent-pir-v2/src/shared_store.rs new file mode 100644 index 0000000..e12c2ed --- /dev/null +++ b/crates/z-transparent-pir-v2/src/shared_store.rs @@ -0,0 +1,113 @@ +use crate::memory_store::MemoryStore; +use crate::store::*; +use std::{cell::RefCell, rc::Rc}; +#[derive(Clone)] +pub(crate) struct SharedStore { + pub inner: Rc>, + pub rewind: Rc>>, +} +impl WalletStore for SharedStore { + fn set_identity(&self) -> Result, StoreError> { + self.inner.borrow().set_identity() + } + fn bind_set(&mut self, identity: &SetIdentity) -> Result<(), StoreError> { + self.inner.borrow_mut().bind_set(identity) + } + fn anchor(&self) -> Result, StoreError> { + self.inner.borrow().anchor() + } + fn scripts(&self) -> Result, StoreError> { + self.inner.borrow().scripts() + } + fn add_scripts(&mut self, entries: &[ScriptEntry]) -> Result { + self.inner.borrow_mut().add_scripts(entries) + } + fn coverage(&self, script: &[u8]) -> Result, StoreError> { + self.inner.borrow().coverage(script) + } + fn provisional(&self) -> Result, StoreError> { + self.inner.borrow().provisional() + } + fn events(&self) -> Result, StoreError> { + self.inner.borrow().events() + } + fn commit_shard(&mut self, commit: ShardCommit) -> Result { + self.inner.borrow_mut().commit_shard(commit) + } + fn commit_anchor( + &mut self, + anchor: &Anchor, + settled_through: u64, + covered_through: u64, + ) -> Result { + self.inner + .borrow_mut() + .commit_anchor(anchor, settled_through, covered_through) + } + fn rollback_above(&mut self, anchor: &Anchor, reason: &str) -> Result { + let result = self.inner.borrow_mut().rollback_above(anchor, reason)?; + let mut rewind = self.rewind.borrow_mut(); + if rewind + .as_ref() + .is_none_or(|held| anchor.height < held.height) + { + *rewind = Some(anchor.clone()); + } + Ok(result) + } + fn promote_provisional( + &mut self, + shard_id: u64, + revision_digest: &str, + ) -> Result<(), StoreError> { + self.inner + .borrow_mut() + .promote_provisional(shard_id, revision_digest) + } + fn pending(&self) -> Result, StoreError> { + self.inner.borrow().pending() + } + fn pending_limit(&self) -> usize { + self.inner.borrow().pending_limit() + } + fn cached_run( + &self, + child_digest: &str, + script: &[u8], + ) -> Result, StoreError> { + self.inner.borrow().cached_run(child_digest, script) + } + fn setup(&self, key: &SetupKey) -> Result, StoreError> { + self.inner.borrow().setup(key) + } + fn put_setup(&mut self, key: &SetupKey, blob: &SetupBlob) -> Result<(), StoreError> { + self.inner.borrow_mut().put_setup(key, blob) + } + fn drop_setup(&mut self, key: &SetupKey) -> Result<(), StoreError> { + self.inner.borrow_mut().drop_setup(key) + } + fn filter( + &self, + revision_digest: &str, + filter_hash: &str, + ) -> Result>, StoreError> { + self.inner.borrow().filter(revision_digest, filter_hash) + } + fn put_filter( + &mut self, + revision_digest: &str, + filter_hash: &str, + sealed: bool, + bytes: &[u8], + ) -> Result<(), StoreError> { + self.inner + .borrow_mut() + .put_filter(revision_digest, filter_hash, sealed, bytes) + } + fn last_commit(&self) -> Result { + self.inner.borrow().last_commit() + } + fn entry_failures(&self, shard_id: u64, script: &[u8]) -> Result { + self.inner.borrow().entry_failures(shard_id, script) + } +} diff --git a/crates/z-transparent-pir-v2/src/store.rs b/crates/z-transparent-pir-v2/src/store.rs new file mode 100644 index 0000000..bae9b6a --- /dev/null +++ b/crates/z-transparent-pir-v2/src/store.rs @@ -0,0 +1,555 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! What a wallet keeps between syncs, and the one boundary it changes at. +//! +//! A sync that starts from nothing every time cannot be a returning wallet: it +//! forgets the receive it found last month, so the spend it finds today is +//! unresolved, and moving the birthday forward to save work throws the receive +//! away for good. The store is the wallet's memory of what it has recovered, +//! which scripts it is responsible for and over which ranges each is covered, +//! what it accepted as the chain's anchor, and what private work it still owes. +//! +//! # One commit per shard +//! +//! Everything a shard yields is committed together — its events, the coverage +//! it establishes for each script, and the page work still owed — or not at +//! all. A crash between two commits loses at most one shard's retrieval, which +//! the next sync repeats; it never records coverage for events it did not keep +//! or events under coverage it did not record. Retrying a commit is idempotent +//! by construction: events are keyed by their stable identities, coverage by +//! script and range, and a repeat that *differs* in any field is a +//! contradiction, refused rather than merged. +//! +//! # Provisional and settled +//! +//! Coverage taken from an unsealed tail carries that revision's digest. When +//! the tail is superseded, that coverage and the events under it are truncated +//! and the range re-derived; when the same digest is later sealed, it is +//! promoted in place. Settled coverage is never rewritten except by a reorg, +//! which rolls everything above the accepted ancestor back at once. +//! +//! # What the wallet owns +//! +//! This is a trait so the wallet that embeds the library supplies the storage: +//! its own database, its own transaction, its own encryption. The reference +//! implementations — in memory here, SQLite in `transparent-wallet-store` — +//! are what the tests run against and what a first integration can use as-is. + +use crate::client::Table; +use crate::ledger::{Ledger, LedgerError}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use transparent_events::TransparentEvent; +use transparent_filter::{SealParameters, ShardMap}; + +/// The publication lineage a store is bound to. +/// +/// Everything in the map that does not change as the set grows: chain, +/// profile, envelope version, first height, and the seal thresholds each +/// geometry was sealed under. The map digest itself changes with every tail +/// republication and is deliberately not part of this. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SetIdentity { + pub shard_schema: String, + pub network: String, + pub genesis_hash: String, + pub profile: String, + pub range_envelope_version: u16, + pub start_height: u64, + pub seal: BTreeMap, +} + +impl SetIdentity { + pub fn of_schema(map: &ShardMap, schema: &str) -> Self { + Self { + shard_schema: schema.to_string(), + network: map.network.clone(), + genesis_hash: map.genesis_hash.clone(), + profile: map.profile.clone(), + range_envelope_version: map.range_envelope_version, + start_height: map.start_height, + seal: map.seal.clone(), + } + } + + /// SHA-256 of the canonical serialization. + pub fn digest(&self) -> String { + hex::encode(Sha256::digest( + serde_json::to_vec(self).expect("a set identity serializes"), + )) + } + + /// Whether `other` is the same lineage, allowing a geometry the store had + /// not seen to appear (a set growing into another tier) but never a + /// changed seal for one already in use. + pub fn continues(&self, other: &Self) -> bool { + self.shard_schema == other.shard_schema + && self.network == other.network + && self.genesis_hash == other.genesis_hash + && self.profile == other.profile + && self.range_envelope_version == other.range_envelope_version + && self.start_height == other.start_height + && self + .seal + .iter() + .all(|(name, seal)| other.seal.get(name).is_none_or(|theirs| theirs == seal)) + } +} + +/// The chain block a wallet has accepted as the end of its coverage. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Anchor { + pub height: u64, + /// Display hex. + pub hash: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum ScriptOrigin { + /// From the wallet's own derivation rules. + Derived, + /// Imported by the user; its history may begin anywhere. + Imported, +} + +/// One script the wallet is responsible for, and the height its required +/// coverage begins at. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ScriptEntry { + pub script: Vec, + pub origin: ScriptOrigin, + /// First height the wallet needs covered for this script. The wallet's + /// own rule decides it; a derived script's is typically the birthday, + /// an imported script's is the set's first height. + pub required_from: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum CoverageKind { + /// From a sealed shard. Immutable short of a reorg. + Settled, + /// From an unsealed tail revision. Replaced when the tail is republished. + Provisional, +} + +/// One script's coverage over one shard's range. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CoverageRange { + pub script: Vec, + pub start_height: u64, + pub end_height: u64, + pub kind: CoverageKind, + pub shard_id: u64, + pub revision_digest: String, + pub terminal_block_hash: String, + /// Full published endpoint, distinct from the covered endpoint. + pub source_anchor: Option, +} + +/// One event the wallet keeps, with where it came from. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StoredEvent { + pub script: Vec, + #[serde(with = "crate::serialization::event")] + pub event: TransparentEvent, + pub shard_id: u64, + pub revision_digest: String, +} + +/// Last committed fragment's boundary, needed to validate greedy packing after resume. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PageBoundary { + pub event_bytes: u32, + #[serde(with = "crate::serialization::event")] + pub last_event: TransparentEvent, +} + +/// Where a chain walk resumes: the run it reads next. +/// +/// A script's history in a directory shard is a chain of runs, newest first, +/// each in an earlier page shard than the one before. The directory entry +/// names the newest; every run's first fragment names the next older one. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RunCursor { + /// The page shard `j` holding the run; its table is `pages-`. + pub child: u8, + /// Zero-based row of the run's first fragment in that table. + pub row: u32, + /// How many runs the run's first fragment must report as older than it: + /// one fewer than the newer run reported. `None` for the newest run, + /// which the directory entry names without a count. + pub older_runs: Option, +} + +/// Page retrieval a wallet still owes for one script in one directory shard +/// revision: the rest of its chain of runs. +/// +/// Every run newer than the cursor has been read, checked and committed. Kept +/// durably so an exhausted budget or an outage leaves resumable work, not a +/// synchronized balance. Keyed by the directory shard revision alone, whose +/// digest commits to every page shard. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PendingPages { + /// Assigned by the store; `None` until first committed. + pub id: Option, + pub shard_id: u64, + pub revision_digest: String, + pub script: Vec, + /// The run the walk reads next. + pub cursor: RunCursor, + /// The two oldest events already read, oldest first: the newer runs' or, + /// before any run was read, the directory entry's inline events. The run + /// at the cursor must be older than both, and each of its events was + /// pushed out by the event two places after it, which these supply for + /// its last two. + #[serde(with = "crate::serialization::events")] + pub carried: Vec, + /// Checked fragments of the current run, each encoded as a page row holding + /// only this script. They are revalidated on resume and cleared at the next run. + pub fragments: Vec>, + pub attempts: u32, + pub target_anchor: Option, +} + +/// One script's run in a sealed page shard, as a wallet verified it, and the +/// name of the cached run below it. +/// +/// Keyed by the page shard's chained identity +/// ([`ShardManifest::child_digests`](transparent_shard::ShardManifest::child_digests)), +/// which covers that page shard and every older one in its directory shard, so +/// nothing the run or the runs below it depend on can change under the key. +/// Following `older` back to the start of the directory shard gives the +/// script's history there, and a walk that reaches a cached run stops at it. +/// +/// A store drops a directory shard's cached runs for a script once it covers +/// that script over the shard sealed: no later walk of the shard is expected. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CachedRun { + /// The page shard's chained identity, in hex. + pub child_digest: String, + pub script: Vec, + /// The directory shard the run belongs to. + pub shard_id: u64, + /// The run's own events, oldest first. + #[serde(with = "crate::serialization::events")] + pub events: Vec, + /// The chained identity of the page shard holding the script's previous + /// run, or `None` when this is its oldest run in the directory shard. + pub older: Option, +} + +/// What one shard's retrieval commits, all at once. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ShardCommit { + /// Original publication endpoint when coverage is clipped to a wallet anchor. + pub source_anchor: Option, + pub shard_id: u64, + pub revision_digest: String, + pub sealed: bool, + pub start_height: u64, + pub end_height: u64, + pub terminal_block_hash: String, + pub events: Vec, + /// Scripts whose coverage this commit extends over the shard's range. + pub covered_scripts: Vec>, + /// Pending page work created or advanced by this commit. + pub pending_upsert: Vec, + /// Pending ids this commit finishes. + pub pending_complete: Vec, + /// Verified runs to keep for later walks. + pub cached_runs: Vec, + /// Scripts whose directory entry in this shard failed on content in this + /// commit's retrieval: a reply that did not decode or verify, or a record + /// that contradicts what the store holds. Each adds one to the entry's + /// [`WalletStore::entry_failures`]; unlike everything else here, that is + /// not idempotent, so a commit carrying failures is made once. + pub failed_entries: Vec>, +} + +impl ShardCommit { + /// Whether this commit covers its `covered_scripts` over the whole of a + /// sealed shard, so no later walk of the shard is expected for them and + /// the store drops their cached runs in it. Coverage clipped to a wallet + /// anchor inside the shard does not: the next sync carries on through it. + pub fn finishes_shard(&self) -> bool { + self.sealed + && self + .source_anchor + .as_ref() + .is_none_or(|source| source.height == self.end_height) + } +} + +/// Published setup for one table segment, kept for reuse. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SetupBlob { + pub public_params_base64: String, + pub public_params_sha256: String, +} + +/// What a setup is keyed by: the set lineage, the exact revision, the table +/// and the segment. Anything less would reuse parameters against bytes they +/// were not derived from. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct SetupKey { + pub set_digest: String, + pub revision_digest: String, + #[serde(with = "crate::serialization::table")] + pub table: Table, + pub segment: u32, +} + +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + #[error("store is corrupt: {0}")] + Corrupt(String), + /// A repeat that differs. Nothing is written. + #[error("contradiction: {0}")] + Contradiction(#[from] LedgerError), + #[error("the store is bound to set {stored}, this map is {offered}")] + SetMismatch { stored: String, offered: String }, + #[error("the commit would leave more than {limit} pending page retrievals")] + PendingLimit { limit: usize }, +} + +/// The wallet's durable memory. See the module documentation. +pub trait WalletStore { + /// The lineage this store is bound to, with its digest, if any. + fn set_identity(&self) -> Result, StoreError>; + /// Binds a fresh store to a lineage, or confirms a bound one continues it. + fn bind_set(&mut self, identity: &SetIdentity) -> Result<(), StoreError>; + + fn anchor(&self) -> Result, StoreError>; + + fn scripts(&self) -> Result, StoreError>; + /// Adds scripts. An existing script's `required_from` is never raised: + /// coverage once required stays required. + fn add_scripts(&mut self, entries: &[ScriptEntry]) -> Result; + + fn coverage(&self, script: &[u8]) -> Result, StoreError>; + /// Every provisional range across every script. + fn provisional(&self) -> Result, StoreError>; + + /// Every kept event in canonical order. + fn events(&self) -> Result, StoreError>; + /// The ledger replayed from `events`. + fn ledger(&self) -> Result { + let mut events: Vec<(Vec, TransparentEvent)> = self + .events()? + .into_iter() + .map(|stored| (stored.script, stored.event)) + .collect(); + let mut ledger = Ledger::new(); + ledger.replay(&mut events)?; + Ok(ledger) + } + + /// The one mutating boundary for retrieval. Atomic and idempotent; a + /// repeat that differs is refused whole. + fn commit_shard(&mut self, commit: ShardCommit) -> Result; + /// Records the accepted anchor and derived coverage summaries. + fn commit_anchor( + &mut self, + anchor: &Anchor, + settled_through: u64, + covered_through: u64, + ) -> Result; + /// Removes every event, coverage range and pending item above `height` + /// and lowers the anchor to it. A reorg or a replaced provisional tail. + fn rollback_above(&mut self, anchor: &Anchor, reason: &str) -> Result; + /// Provisional coverage under `revision_digest` becomes settled: the same + /// bytes were sealed. + fn promote_provisional( + &mut self, + shard_id: u64, + revision_digest: &str, + ) -> Result<(), StoreError>; + + fn pending(&self) -> Result, StoreError>; + /// The run cached for `script` in the page shard with chained identity + /// `child_digest`, if any. See [`CachedRun`]. + fn cached_run( + &self, + child_digest: &str, + script: &[u8], + ) -> Result, StoreError>; + /// The most pending page retrievals a commit may leave behind. + fn pending_limit(&self) -> usize; + + fn setup(&self, key: &SetupKey) -> Result, StoreError>; + fn put_setup(&mut self, key: &SetupKey, blob: &SetupBlob) -> Result<(), StoreError>; + /// Forgets a setup a reply showed to be stale, so the next sync fetches + /// the published one again. Forgetting one not held is not an error. + fn drop_setup(&mut self, key: &SetupKey) -> Result<(), StoreError>; + fn filter( + &self, + revision_digest: &str, + filter_hash: &str, + ) -> Result>, StoreError>; + fn put_filter( + &mut self, + revision_digest: &str, + filter_hash: &str, + sealed: bool, + bytes: &[u8], + ) -> Result<(), StoreError>; + + /// The id of the last commit, zero before any. + fn last_commit(&self) -> Result; + + /// Content failures recorded for `script`'s directory entry in shard + /// `shard_id`, across every sync: the number of commits that named it in + /// [`ShardCommit::failed_entries`]. Never reset by a later success, a + /// new revision of the shard or a rollback, so the cap on attempts holds + /// whatever the service does in between. + fn entry_failures(&self, shard_id: u64, script: &[u8]) -> Result; +} + +impl WalletStore for Box { + fn set_identity(&self) -> Result, StoreError> { + (**self).set_identity() + } + fn bind_set(&mut self, identity: &SetIdentity) -> Result<(), StoreError> { + (**self).bind_set(identity) + } + fn anchor(&self) -> Result, StoreError> { + (**self).anchor() + } + fn scripts(&self) -> Result, StoreError> { + (**self).scripts() + } + fn add_scripts(&mut self, entries: &[ScriptEntry]) -> Result { + (**self).add_scripts(entries) + } + fn coverage(&self, script: &[u8]) -> Result, StoreError> { + (**self).coverage(script) + } + fn provisional(&self) -> Result, StoreError> { + (**self).provisional() + } + fn events(&self) -> Result, StoreError> { + (**self).events() + } + fn commit_shard(&mut self, commit: ShardCommit) -> Result { + (**self).commit_shard(commit) + } + fn commit_anchor( + &mut self, + anchor: &Anchor, + settled_through: u64, + covered_through: u64, + ) -> Result { + (**self).commit_anchor(anchor, settled_through, covered_through) + } + fn rollback_above(&mut self, anchor: &Anchor, reason: &str) -> Result { + (**self).rollback_above(anchor, reason) + } + fn promote_provisional( + &mut self, + shard_id: u64, + revision_digest: &str, + ) -> Result<(), StoreError> { + (**self).promote_provisional(shard_id, revision_digest) + } + fn pending(&self) -> Result, StoreError> { + (**self).pending() + } + fn pending_limit(&self) -> usize { + (**self).pending_limit() + } + fn cached_run( + &self, + child_digest: &str, + script: &[u8], + ) -> Result, StoreError> { + (**self).cached_run(child_digest, script) + } + fn setup(&self, key: &SetupKey) -> Result, StoreError> { + (**self).setup(key) + } + fn put_setup(&mut self, key: &SetupKey, blob: &SetupBlob) -> Result<(), StoreError> { + (**self).put_setup(key, blob) + } + fn drop_setup(&mut self, key: &SetupKey) -> Result<(), StoreError> { + (**self).drop_setup(key) + } + fn filter( + &self, + revision_digest: &str, + filter_hash: &str, + ) -> Result>, StoreError> { + (**self).filter(revision_digest, filter_hash) + } + fn put_filter( + &mut self, + revision_digest: &str, + filter_hash: &str, + sealed: bool, + bytes: &[u8], + ) -> Result<(), StoreError> { + (**self).put_filter(revision_digest, filter_hash, sealed, bytes) + } + fn last_commit(&self) -> Result { + (**self).last_commit() + } + fn entry_failures(&self, shard_id: u64, script: &[u8]) -> Result { + (**self).entry_failures(shard_id, script) + } +} + +/// The identity a receive is keyed by. +pub fn receive_key(event: &transparent_events::ReceiveEvent) -> (transparent_events::Txid, u32) { + (event.txid, event.output_index) +} + +/// The identity a spend is keyed by. +pub fn spend_key( + event: &transparent_events::SpendEvent, +) -> (transparent_events::Txid, u32, transparent_events::Txid, u32) { + ( + event.spending_txid, + event.input_index, + event.spent_txid, + event.spent_output_index, + ) +} + +/// Sorts one script's ranges and drops exact repeats. +/// +/// Ranges are kept per shard rather than merged into runs, deliberately: each +/// carries the block hash its coverage rests on, and a reorg is found by +/// asking the wallet's chain about those hashes newest first. Merging shards +/// 0-2 into one range would keep only shard 2's hash, and a reorg inside +/// shard 1 would then roll back to the set's start instead of to shard 0. +pub fn merge_coverage(mut ranges: Vec) -> Vec { + ranges.sort_by_key(|range| (range.start_height, range.end_height)); + ranges.dedup(); + ranges +} + +/// The heights in `[from, through]` that `ranges` do not cover, as inclusive +/// gaps in ascending order. +pub fn uncovered(ranges: &[CoverageRange], from: u64, through: u64) -> Vec<(u64, u64)> { + let mut sorted: Vec<&CoverageRange> = ranges.iter().collect(); + sorted.sort_by_key(|range| range.start_height); + let mut gaps = Vec::new(); + let mut next = from; + for range in sorted { + if range.end_height < next { + continue; + } + if range.start_height > next { + gaps.push((next, range.start_height.min(through + 1) - 1)); + } + next = next.max(range.end_height + 1); + if next > through { + break; + } + } + if next <= through { + gaps.push((next, through)); + } + gaps.retain(|(a, b)| a <= b); + gaps +} diff --git a/crates/z-transparent-pir-v2/src/sync.rs b/crates/z-transparent-pir-v2/src/sync.rs new file mode 100644 index 0000000..3f3d40b --- /dev/null +++ b/crates/z-transparent-pir-v2/src/sync.rs @@ -0,0 +1,3901 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! Birthday to balance: the join between public filters and private retrieval. +//! +//! This is the path the whole design exists to make cheap: +//! +//! 1. fetch the published shard map and binary-search it for the birthday; +//! 2. download **every** filter from that shard forward and match locally; +//! 3. for each matched script and directory shard, privately retrieve its +//! directory row, then walk the chain of runs its entry points to, newest +//! first, through the shard's page shards (see `walk_chains`); +//! 4. replay the events into a UTXO set. +//! +//! Step 2 downloads every filter in range, not only the ones that will match. +//! Which filters a wallet asks for must not be a function of its scripts. +//! +//! Step 3 addresses a row over the shard's whole logical row space, which is +//! its segments concatenated. Every segment answers, and the row is identified +//! by its exact script bytes rather than by which segment returned it: asking +//! only the segment that holds the script would disclose part of the script. +//! Coverage advances for a shard only once every segment has been processed. +//! +//! # Settled and provisional coverage +//! +//! The last shard in the map may be a provisional tail revision, which will be +//! superseded as the chain advances. Coverage taken from one is reported +//! separately from settled coverage, with the revision's digest, so a caller +//! can re-derive that range instead of merging a later revision into it. +//! +//! # What the checks here establish +//! +//! The first profile trusts the publisher and the retrieval service to provide +//! correct, complete, mutually consistent data for the declared anchor. What +//! this code checks — filter digests against the map, exact script bytes +//! against the row, a page header against the entry that located it, the event +//! total against what the pages yielded — detects corruption, stale data and +//! accidentally mixed shards. None of it establishes that the index matches the +//! chain: a service that returns a consistent lie passes every one of them. +//! +//! # What this leaks, stated plainly +//! +//! Step 3 queries only the shards that matched, so the service learns which +//! chain ranges the wallet had probable activity in. That is the leak the +//! design accepts, and it is real: with content-sealed shards a busy period is +//! covered by a narrower shard, so the localization is sharper during busy +//! periods than quiet ones. This code does not hide it and must not be +//! described as if it did. + +use crate::adapters::{Acceptance, ChainView, ScriptProvider}; +use crate::client::{classify_transport, ClientError, Table, TableClient}; +use crate::ledger::{Ledger, LedgerError}; +use crate::store::{ + uncovered, Anchor, CachedRun, CoverageKind, CoverageRange, PendingPages, RunCursor, + SetIdentity, SetupBlob, SetupKey, ShardCommit, StoreError, StoredEvent, WalletStore, +}; +use crate::transport::ShardRequest; +use crate::transport::{ByteCharges, FilterSource, ShardTransport, StaleRevision}; +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap}; +use transparent_events::TransparentEvent; +use transparent_filter::{BlockHash, FilterLimits, ScriptBytes, ShardKey, ShardMap}; +use transparent_shard::build::candidate_rows; +use transparent_shard::manifest::{PageChild, ShardManifest}; +use transparent_shard::page_row::PageEntry; +use transparent_shard::records::DirectoryEntry; + +#[path = "sync_ahead.rs"] +mod ahead; + +#[derive(Debug, thiserror::Error)] +pub enum SyncError { + #[error("client: {0}")] + Client(#[from] ClientError), + #[error("ledger: {0}")] + Ledger(#[from] LedgerError), + #[error("filter: {0}")] + Filter(#[from] transparent_filter::FilterError), + #[error("record: {0}")] + Record(#[from] transparent_shard::records::RecordError), + #[error("transport: {0}")] + Transport(String), + #[error("store: {0}")] + Store(#[from] StoreError), + /// The store's accepted anchor lies above the offered map's end, and the + /// wallet's chain still accepts it. A lower anchor is never accepted + /// silently. + #[error("the store's anchor {stored} is above the map's end {offered}, which the chain still accepts")] + AnchorRegressed { stored: u64, offered: u64 }, + #[error("{0}")] + Invalid(String), + /// A row in a private reply failed on content: it did not decode, a + /// record or link in it was not what the shard declares, or it + /// contradicts what the store holds. A reply failing as a whole (its + /// length, revision binding or parameter epoch) is a request failure + /// instead, [`SyncError::Client`]: the service makes those without + /// knowing the row. + /// + /// A sync does not stop for one: it fails the directory entry the reply + /// was for, finishes everything else, counts the failure in the store + /// and reports [`IncompleteReason::ContentFailed`]; see + /// [`retrieve_shard_into`] and [`walk_chains`]. It reaches a caller from + /// the one-shot [`sync`], which has no store to count it in, and when a + /// contradiction cannot be laid at one entry's door. + #[error("content: {0}")] + Content(String), + #[error("service serves schema {served}, this build reads {expected}")] + Schema { + served: String, + expected: &'static str, + }, + /// A shard names a geometry this build has no parameters for. + /// + /// Fatal for the whole sync rather than a reason to skip the shard. Skipping + /// would advance coverage past a range whose history was never retrieved, + /// and the wallet would report a synchronised balance it has not earned. + #[error( + "shard set uses geometry {0}, which this build does not know; upgrade before syncing this \ + range" + )] + UnknownGeometry(String), + /// The map names a range-filter profile this build does not know. + /// + /// Its filters would be decoded under the wrong Golomb-Rice parameters, + /// which neither fails reliably nor matches correctly, so the sync stops + /// before matching anything. + #[error( + "shard set uses range-filter profile {0}, which this build does not know; upgrade \ + before syncing this range" + )] + UnknownProfile(String), + /// A revision this sync needed was withdrawn, and refreshing the map did + /// not offer one that is served. + /// + /// Terminal, and coverage stopped where it was. Distinct from a transport + /// failure so a caller can re-validate a map against its accepted chain and + /// come back, rather than treating a routine republication as a broken set. + #[error( + "shard {shard_id}: {stale}; the map was refreshed {refreshes} time(s) without offering a \ + live revision" + )] + StaleRevision { + shard_id: u64, + stale: StaleRevision, + refreshes: u32, + }, + /// A shard's manifest does not describe the shard the map named. + /// + /// Terminal for the sync, and coverage stops where it was. Every field a + /// retrieval relies on — geometry, range, segment shape, chain link — is + /// checked against the map before a single private request, so a + /// publication that is internally inconsistent is refused rather than read + /// under whichever of its two stories the wallet happened to see first. + #[error("shard {shard_id}: manifest {field} does not agree with the map")] + ManifestMismatch { shard_id: u64, field: &'static str }, + /// A refreshed map is not the same shard set, continued. + /// + /// The caller validated the map this sync started from against the wallet's + /// accepted chain. A map fetched mid-sync inherits that validation only over + /// the range the two agree on, so a disagreement about already-covered + /// history is a different set rather than a longer one, and the caller has + /// to re-validate before retrying. + #[error( + "the refreshed shard map is not a continuation of the one this sync started from: {0}" + )] + MapDiverged(String), + /// The map contradicts sealed history the store holds on the wallet's own + /// chain, and declares no re-cut of it. + /// + /// Sealed content is immutable. A sealed range described differently is a + /// reorg when the chain has moved, and is rolled back as one. This is the + /// other case: the chain accepts the block the stored range rests on and + /// also the block the map's sealed shard now covering its start ends on, + /// at or below the target. Both are on the chain the wallet follows, so + /// nothing explains the change but the publisher, and retrying does not + /// resolve it while the publisher serves this history. When the chain + /// does not yet accept the replacement's end block, as when the publisher + /// followed a shallow reorg through a shard it had just sealed before the + /// wallet's chain did, the sync stops with + /// [`IncompleteReason::ChainUnknown`] instead and becomes an ordinary reorg + /// once the chain moves. Refused before anything is read, and after + /// rolling back only a reorg the wallet's chain itself shows, which runs + /// first so that no map can hold it back. Distinct from + /// [`MapDiverged`](Self::MapDiverged), which a re-cut published mid-sync + /// also produces and which the next sync resolves. + /// + /// A replica still serving a map from before a re-cut the store has + /// already followed looks the same from here, since the store keeps no + /// re-cut epoch: an embedding that records the epoch it last synced at + /// can tell the two apart before calling the sync. + #[error( + "the map replaces the sealed shard the store holds from height {start_height} (revision \ + {revision_digest}) with another the wallet's chain also accepts, and declares no re-cut \ + of it" + )] + SealedRewrite { + start_height: u64, + revision_digest: String, + }, +} + +/// Map refreshes one sync will spend recovering a withdrawn revision. +/// +/// A tail is republished on a block cadence, so a wallet lapped this many times +/// while reading is not converging on the tip, and saying so beats re-reading a +/// moving target indefinitely. Every refresh must also *change* the refused +/// revision, so this bounds genuine republication rather than a loop. +const MAX_MAP_REFRESHES: u32 = 4; + +/// Passes a sync makes while the wallet's rules keep adding scripts. A +/// gap-limit advance that never converges is a wallet fault; this stops it +/// from becoming an unbounded sync. +const MAX_DISCOVERY_PASSES: u32 = 16; + +/// Content failures after which a directory entry is no longer queried. +/// +/// A service can aim a bad reply at chosen rows and watch what the wallet +/// does next. Each failure changes later traffic once (the entry is asked +/// again, then not), so this is how many such observations an entry gives +/// away; a service could deny the entry service anyway by corrupting it. +pub const MAX_CONTENT_FAILURES: u32 = 3; + +/// One geometry's parameters, as a service declares them. +/// +/// The dimensions come with the schemes so a wallet can check the pair rather +/// than adopt either. A service naming a registry geometry but publishing other +/// row counts under it is refused; so is one publishing the registry's row +/// counts with a scheme they do not derive. +#[allow(dead_code)] // Published compatibility fields are retained for source parity. +pub struct GeometryParams { + pub name: String, + pub directory_rows: u64, + pub directory_row_bytes: u32, + pub directory_scheme: transparent_native::NativeScheme, + /// The 44-bit dithered directory scheme, when the service advertises one. + pub directory_scheme_dq44: Option, + pub directory_setup_seed: u64, + pub page_rows: u64, + pub page_row_bytes: u32, + pub pages_scheme: transparent_native::NativeScheme, + /// The 44-bit dithered page scheme, when the service advertises one. + pub pages_scheme_dq44: Option, + pub pages_setup_seed: u64, +} + +/// The geometries a service declares, checked against this build's registry. +pub struct ServiceGeometry { + /// The schema the service serves, from its init response. + /// + /// A row's meaning is entirely a function of its schema, and the bytes + /// carry no version of their own — a fixed-width row has nowhere to put + /// one without spending space on every row. So a wallet that decoded + /// whatever it was handed would read a newer layout as the one it knows + /// and reconstruct a plausible, wrong history. This is where that stops. + pub schema: String, + /// One entry per geometry the service holds. A set spanning an archive tier + /// and a recent tier declares two. + pub geometries: Vec, +} + +/// The prepared clients for one geometry: one per table. +struct GeometryClients { + directory: TableClient, + pages: TableClient, +} + +/// Every geometry a sync has prepared, by registry name. +/// +/// Prepared on first use rather than up front, so a wallet syncing only recent +/// history never pays to derive archive parameters. Once prepared, a geometry's +/// clients are reused for every shard naming it — which is the saving that +/// naming geometries from a closed registry exists to buy. +struct Clients { + schema: String, + prepared: HashMap, +} + +impl Clients { + /// Prepares the clients for `name`, checking the service's declaration + /// against this build's registry and re-deriving the scheme. + /// + /// Three things have to agree: the name must be one this build knows, the + /// dimensions the service published under it must be the registry's, and + /// the scheme must be what those dimensions derive. A service that could + /// move any one of them independently could choose a geometry for the + /// wallet, and choosing the geometry is choosing what the response reveals. + fn prepare( + &mut self, + name: &str, + declared: &[GeometryParams], + ) -> Result<&mut GeometryClients, SyncError> { + if !self.prepared.contains_key(name) { + let geometry = transparent_shard::layout::by_name(name) + .ok_or_else(|| SyncError::UnknownGeometry(name.to_string()))?; + let params = declared + .iter() + .find(|params| params.name == name) + .ok_or_else(|| { + SyncError::Invalid(format!( + "the map uses geometry {name} but the service declares no parameters for it" + )) + })?; + if params.directory_rows != geometry.directory_rows + || params.directory_row_bytes != geometry.directory_row_bytes as u32 + || params.page_rows != geometry.page_rows + || params.page_row_bytes != geometry.page_row_bytes as u32 + { + return Err(SyncError::Invalid(format!( + "the service declares geometry {name} with dimensions this build does not \ + know it by" + ))); + } + let clients = GeometryClients { + directory: TableClient::new_with_schema( + &self.schema, + Table::Directory, + name, + geometry.directory_rows, + geometry.directory_row_bytes as u32, + ¶ms.directory_scheme, + params.directory_scheme_dq44.as_ref(), + )?, + pages: TableClient::new_with_schema( + &self.schema, + Table::Pages(0), + name, + geometry.page_rows, + geometry.page_row_bytes as u32, + ¶ms.pages_scheme, + params.pages_scheme_dq44.as_ref(), + )?, + }; + self.prepared.insert(name.to_string(), clients); + } + Ok(self.prepared.get_mut(name).expect("just prepared")) + } +} + +/// One provisional revision a sync took coverage from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProvisionalCoverage { + pub shard_id: u64, + pub revision: u32, + pub manifest_digest: String, + pub end_height: u64, +} + +/// Bounds on one call to [`sync_into`]. `None` is unbounded. +/// +/// Reaching a bound is not an error: the sync commits what it has, records +/// the page work still owed, and reports `Incomplete`. The next call resumes +/// from the durable pending work. +#[derive(Clone, Copy, Debug, Default)] +pub struct WorkLimits { + /// Private queries, directory and pages together. + pub max_queries: Option, + /// Private bytes: setup, query uploads and responses. + pub max_private_bytes: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum IncompleteReason { + QueryBudget, + ByteBudget, + /// The store would not hold more pending page work. + PendingLimit, + /// The service stayed at capacity for the whole retry budget on this shard. + Overloaded { + shard_id: u64, + }, + /// The wallet's chain view could not confirm a block its coverage rests + /// on, so neither a reorg nor its absence could be established. + ChainUnknown { + height: u64, + }, + /// The wallet's rules kept adding scripts past the pass bound. + DiscoveryUnbounded, + /// Coverage stops short at directory entries whose replies failed on + /// content; [`SyncReport::failed_entries`] lists them. Everything else + /// was read and committed. + ContentFailed, + PublicationBehind { + height: u64, + }, + UnresolvedSpends, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Completion { + Complete, + Incomplete { + reason: IncompleteReason, + /// Pending page retrievals left in the store. + pending: usize, + }, +} + +/// What one [`sync_into`] call did. +#[allow(dead_code)] // Internal parity report, never a product balance API. +pub struct SyncReport { + /// The store's whole ledger after this sync, not only what it added. + pub ledger: Ledger, + pub charges: ByteCharges, + pub matched_shards: Vec, + pub unproductive_matches: u64, + /// Height through which every script in scope is covered from its + /// required height, provisional coverage included. + pub covered_through: u64, + /// The same counting settled coverage only. + pub settled_through: u64, + pub provisional: Vec, + pub map_refreshes: u32, + pub completion: Completion, + /// The height everything above was rolled back to, if this sync found a + /// reorg or a replaced provisional tail. + pub rolled_back_to: Option, + /// Provisional revision digests whose coverage was truncated and + /// re-derived. + pub replaced_revisions: Vec, + /// Scripts the wallet's rules added during this sync. + pub scripts_added: usize, + /// Store commits this sync made. + pub commits: u64, + /// Directory entries this sync failed on content or held back because + /// they had failed [`MAX_CONTENT_FAILURES`] times, by shard and script. + pub failed_entries: Vec, +} + +/// One script's directory entry in one shard that failed on content. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FailedEntry { + pub shard_id: u64, + pub script: Vec, + /// Content failures the store holds for the entry, this sync's included. + /// At [`MAX_CONTENT_FAILURES`] the entry is no longer queried. + pub failures: u32, +} + +/// Syncs a wallet's store to its accepted target, independently of publication tip. +/// +/// See the module documentation for the shape of a sync. The store is the +/// wallet's memory: what it holds decides what is fetched, and what is +/// fetched is committed to it shard by shard. On return the store is +/// consistent whether the sync completed or not. +#[allow(clippy::too_many_arguments)] +pub async fn sync_into( + store: &mut S, + map: &ShardMap, + map_bytes: u64, + geometry: &ServiceGeometry, + chain: &impl ChainView, + scripts: &mut impl ScriptProvider, + filters: &mut impl FilterSource, + transport: &mut impl ShardTransport, + limits: &WorkLimits, + target_anchor: &Anchor, +) -> Result { + if !transparent_shard::manifest::supported_schema(&geometry.schema) { + return Err(SyncError::Schema { + served: geometry.schema.clone(), + expected: transparent_shard::SCHEMA, + }); + } + map.check_shape() + .map_err(|error| SyncError::Invalid(format!("shard map is malformed: {error}")))?; + if transparent_filter::range_profile(&map.profile).is_none() { + return Err(SyncError::UnknownProfile(map.profile.clone())); + } + let genesis = BlockHash::from_display_hex(&map.genesis_hash)?; + BlockHash::from_display_hex(&target_anchor.hash)?; + if target_anchor.height < map.start_height { + return Err(SyncError::Invalid( + "target precedes publication start".into(), + )); + } + match chain.is_accepted(target_anchor.height, &target_anchor.hash) { + Acceptance::Rejected => { + return Err(SyncError::Invalid( + "target anchor rejected by wallet chain".into(), + )) + } + Acceptance::Unknown => { + return incomplete_at( + store, + IncompleteReason::ChainUnknown { + height: target_anchor.height, + }, + ) + } + Acceptance::Accepted => {} + } + if map + .shards + .last() + .is_none_or(|s| s.end_height < target_anchor.height) + { + return incomplete_at( + store, + IncompleteReason::PublicationBehind { + height: target_anchor.height, + }, + ); + } + check_target_hash(map, target_anchor)?; + let retained = store + .events()? + .iter() + .map(|e| event_height(&e.event)) + .max() + .unwrap_or(0); + if let Some(a) = store.anchor()? { + if a.height > target_anchor.height + && chain.is_accepted(a.height, &a.hash) == Acceptance::Accepted + { + return Err(SyncError::AnchorRegressed { + stored: a.height, + offered: target_anchor.height, + }); + } + } + if retained > target_anchor.height { + return Err(SyncError::Invalid( + "target below retained events; explicitly roll back to an accepted anchor first".into(), + )); + } + + // The store is bound to a publication lineage. A map from another + // lineage is refused before anything is read from it. + let identity = SetIdentity::of_schema(map, &geometry.schema); + store.bind_set(&identity).map_err(|error| match error { + StoreError::SetMismatch { stored, offered } => SyncError::MapDiverged(format!( + "the store is bound to set {stored}, this map is {offered}" + )), + other => SyncError::Store(other), + })?; + let first_commit = store.last_commit()?; + + let Some(tip_entry) = map.shards.last() else { + return Err(SyncError::Invalid("the map names no shards".into())); + }; + let through = target_anchor.height; + let _ = tip_entry; + + // A map that ends below the anchor the wallet accepted is not a newer + // publication; if the wallet's chain still accepts that anchor, the + // map is refused rather than silently rewound. + if let Some(anchor) = store.anchor()? { + if anchor.height > through + && chain.is_accepted(anchor.height, &anchor.hash) == Acceptance::Accepted + { + return Err(SyncError::AnchorRegressed { + stored: anchor.height, + offered: through, + }); + } + } + + let mut charges = ByteCharges { + map_bytes, + ..Default::default() + }; + let mut rolled_back_to: Option = None; + let mut replaced_revisions: Vec = Vec::new(); + + // Every revision the map's re-cuts declare, by digest, looked up once. + let declared = map.superseded_index(); + + // Reorg detection over what the store holds. Every distinct block the + // coverage rests on is asked of the wallet's chain, newest first; the + // first rejected one rolls everything above the highest accepted block + // below it back. + let mut rests_on: BTreeMap = BTreeMap::new(); + // Settled ranges the map neither still publishes nor declares superseded. + let mut rewritten: Vec = Vec::new(); + for entry in store.scripts()? { + for range in store.coverage(&entry.script)? { + rests_on + .entry(range.end_height) + .or_insert(range.terminal_block_hash.clone()); + if range.kind == CoverageKind::Settled && !vouches_for(map, &declared, &range) { + rewritten.push(range); + } + } + } + let mut ancestor: Option = None; + let mut unknown: Option = None; + for (height, hash) in rests_on.iter().rev() { + match chain.is_accepted(*height, hash) { + Acceptance::Rejected => { + // Roll back to the highest accepted block below this one. + let mut below = map.start_height.saturating_sub(1); + for (candidate, candidate_hash) in rests_on.range(..height).rev() { + if chain.is_accepted(*candidate, candidate_hash) == Acceptance::Accepted { + below = *candidate; + break; + } + } + ancestor = Some(below); + break; + } + Acceptance::Accepted => break, + Acceptance::Unknown => { + unknown = Some(*height); + } + } + } + if let Some(height) = ancestor { + store.rollback_above(&accepted_at(chain, map, height)?, "reorg")?; + rolled_back_to = Some(height); + } else if let Some(height) = unknown { + if !rests_on.is_empty() { + // Nothing could be confirmed. Reading on would either accept a + // branch the wallet has not seen or roll back on a guess. + return chain_unknown( + store, + map, + target_anchor, + charges, + height, + rolled_back_to, + replaced_revisions, + first_commit, + ); + } + } + + // Sealed content is immutable, so a sealed range the map now describes + // differently, without declaring a re-cut of it, is judged by what the + // wallet's chain says; see `judge_rewrite`. Only after the scan above, + // which depends on the chain alone, has rolled back any reorg the chain + // shows, and only for the ranges that rollback kept: a map cannot hold + // back a reorg by changing history below it. A contradiction on the + // wallet's own chain is refused before anything else is rolled back or + // read; anything the chain cannot settle yet stops the sync as an + // unknown block does. + let mut contradiction: Option<&CoverageRange> = None; + let mut unplaced: Option = None; + for range in rewritten + .iter() + .filter(|range| ancestor.is_none_or(|kept| range.end_height <= kept)) + { + match judge_rewrite(map, range, target_anchor, chain) { + Rewrite::Reorg => {} + Rewrite::Contradiction => { + if contradiction.is_none_or(|held| held.start_height > range.start_height) { + contradiction = Some(range); + } + } + Rewrite::Unsettled(height) => { + unplaced = Some(unplaced.map_or(height, |held| held.max(height))); + } + } + } + if let Some(range) = contradiction { + return Err(SyncError::SealedRewrite { + start_height: range.start_height, + revision_digest: range.revision_digest.clone(), + }); + } + if let Some(height) = unplaced { + return chain_unknown( + store, + map, + target_anchor, + charges, + height, + rolled_back_to, + replaced_revisions, + first_commit, + ); + } + + // Provisional reconciliation. A tail revision the map has moved past is + // truncated and re-derived; one the map has sealed is promoted. + let mut truncate_from: Option = None; + let mut promote: Vec<(u64, String)> = Vec::new(); + // A partially retrieved revision may have events and pending pages but no + // completed coverage yet. Unfinished work under a revision the map no + // longer publishes is dropped, and what it saved is rolled back from + // where the revision began, never from wherever the map now puts its + // shard id: a re-cut gives that id to another range. + let mut dropped: Vec = Vec::new(); + let mut saved: Option> = None; + for pending in store.pending()? { + if map + .shards + .get(pending.shard_id as usize) + .is_some_and(|published| published.manifest_digest == pending.revision_digest) + { + continue; + } + if saved.is_none() { + saved = Some(store.events()?); + } + let lowest_saved = saved + .iter() + .flatten() + .filter(|stored| stored.revision_digest == pending.revision_digest) + .map(|stored| event_height(&stored.event)) + .min(); + let declared = declared.get(pending.revision_digest.as_str()).copied(); + match unfinished(&pending, declared, lowest_saved, chain) { + Unfinished::Rewind(height) => { + truncate_from = Some(truncate_from.map_or(height, |held| held.min(height))); + } + Unfinished::Drop => dropped.extend(pending.id), + } + if !replaced_revisions.contains(&pending.revision_digest) { + replaced_revisions.push(pending.revision_digest); + } + } + for range in store.provisional()? { + match map.shards.get(range.shard_id as usize) { + Some(published) if published.manifest_digest == range.revision_digest => { + if published.sealed { + promote.push((range.shard_id, range.revision_digest.clone())); + } + } + _ => { + truncate_from = Some( + truncate_from + .map_or(range.start_height, |held: u64| held.min(range.start_height)), + ); + if !replaced_revisions.contains(&range.revision_digest) { + replaced_revisions.push(range.revision_digest.clone()); + } + } + } + } + if let Some(start) = truncate_from { + // From the start of the shard now covering that height: at or below + // everything the replaced revision saved, and a boundary the map names + // a block for, so the rollback never depends on the wallet's chain + // holding a hash inside a shard. + let start = map + .shard_for_height(start) + .map_or(start, |entry| entry.start_height.min(start)); + let below = start.saturating_sub(1); + if rolled_back_to.is_none_or(|held| held > below) { + store.rollback_above(&accepted_at(chain, map, below)?, "replaced revision")?; + rolled_back_to = Some(below); + } + } + for (shard_id, digest) in promote { + store.promote_provisional(shard_id, &digest)?; + } + + let discard: Vec = store + .pending()? + .iter() + .filter(|p| { + p.target_anchor.as_ref() != Some(target_anchor) + || p.id.is_some_and(|id| dropped.contains(&id)) + }) + .filter_map(|p| p.id) + .collect(); + if !discard.is_empty() { + store.commit_shard(ShardCommit { + pending_complete: discard, + ..Default::default() + })?; + } + + // Replies a lookahead sends early are held here until the walk asks for + // them; see `ahead`. With nothing stashed both pass straight through. + let transport = &mut ahead::Staged::new(transport); + let filters = &mut ahead::StagedFilters::new(filters); + // A budgeted sync walks in sequence: whether a later shard's queries are + // sent at all depends on page work the walk discovers on the way. + let look_ahead = transport.concurrency() > 1 + && limits.max_queries.is_none() + && limits.max_private_bytes.is_none(); + + // The wallet's script set, with its own required heights. + store.add_scripts(&scripts.scripts())?; + let mut scripts_added = 0usize; + let mut clients = Clients { + schema: geometry.schema.clone(), + prepared: HashMap::new(), + }; + let mut matched_shards: Vec = Vec::new(); + let mut unproductive = 0u64; + let mut refreshes = 0u32; + let mut held_map_digest: Option = None; + let mut active: Cow<'_, ShardMap> = Cow::Borrowed(map); + let mut completion = Completion::Complete; + let mut failures = Failures::default(); + + 'passes: for pass in 0..=MAX_DISCOVERY_PASSES { + if pass == MAX_DISCOVERY_PASSES { + completion = Completion::Incomplete { + reason: IncompleteReason::DiscoveryUnbounded, + pending: store.pending()?.len(), + }; + break; + } + let entries = store.scripts()?; + let mut active_scripts: Vec> = Vec::new(); + + // Pending page work first: it is already located, and finishing it + // is the cheapest coverage available. + let pending = store.pending()?; + let mut by_revision: BTreeMap<(u64, String), Vec> = BTreeMap::new(); + for item in pending { + by_revision + .entry((item.shard_id, item.revision_digest.clone())) + .or_default() + .push(item); + } + for ((shard_id, digest), mut items) in by_revision { + // An entry that failed earlier in this sync, or too often, is + // not walked again; one still owing dummies sends them. + let mut kept = Vec::with_capacity(items.len()); + for item in items { + if !hold(store, &mut failures.entries, shard_id, &item.script)? + || failures.owes(shard_id, &item.script).is_some() + { + kept.push(item); + } + } + items = kept; + if items.is_empty() { + continue; + } + let Some(entry) = active.shards.get(shard_id as usize) else { + continue; + }; + if entry.manifest_digest != digest { + // Its revision is gone, here only after a refresh mid-sync; + // the ordinary planning re-derives the range. + continue; + } + let entry = entry.clone(); + let previous_digest = previous_digest_of(&active, shard_id); + let outcome = with_refusals( + store, + &active, + &entry, + &previous_digest, + geometry, + &mut clients, + transport, + &mut charges, + limits, + async |store, prepared, transport, charges, manifest| { + finish_pages( + store, + &entry, + manifest, + &items, + prepared, + transport, + charges, + limits, + target_anchor, + chain, + &mut failures, + ) + .await + }, + ) + .await; + match outcome { + Ok(Some(stopped)) => { + completion = stopped; + break 'passes; + } + Ok(None) => { + if !matched_shards.contains(&shard_id) { + matched_shards.push(shard_id); + } + let events = store.events()?; + for item in &items { + if holds_history(&events, &item.script, &entry, target_anchor.height) + && !active_scripts.contains(&item.script) + { + active_scripts.push(item.script.clone()); + } + } + } + Err(SyncError::StaleRevision { .. }) => { + // Handled by the ordinary planning below on the next map. + } + Err(other) => return Err(other), + } + } + + // Plan: which shards each script still needs. + let through = target_anchor.height; + let mut work: BTreeMap>> = BTreeMap::new(); + for entry in &entries { + let from = entry.required_from.max(active.start_height); + if from > through { + continue; + } + let coverage = store.coverage(&entry.script)?; + for (gap_start, gap_end) in uncovered(&coverage, from, through) { + for (index, shard) in active.shards.iter().enumerate() { + if shard.end_height < gap_start { + continue; + } + if shard.start_height > gap_end { + break; + } + let scripts = work.entry(index).or_default(); + if !scripts.contains(&entry.script) { + scripts.push(entry.script.clone()); + } + } + } + } + + let mut uncached = Vec::new(); + for index in work.keys() { + let entry = &active.shards[*index]; + if store + .filter(&entry.manifest_digest, &entry.filter_hash)? + .is_none() + { + uncached.push(entry.shard_id); + } + } + if filters.uses_parents() { + charges.filter_bytes += filters + .prepare_parents(&active, &uncached, store) + .await + .map_err(|e| SyncError::Transport(e.to_string()))?; + } else { + // Every uncached filter in the walk is downloaded anyway; overlap + // them rather than paying one round trip each. + filters.prefetch(&uncached); + } + let mut index_iter: Vec = work.keys().copied().collect(); + if look_ahead && !filters.uses_parents() { + ahead::look_ahead( + store, + &active, + genesis, + &index_iter, + &work, + geometry, + &mut clients, + filters, + transport, + target_anchor, + chain, + &failures.entries, + ) + .await?; + } + let mut position = 0usize; + while position < index_iter.len() { + let index = index_iter[position]; + let Some(entry) = active.shards.get(index).cloned() else { + break; + }; + let scripts_here = work.get(&index).cloned().unwrap_or_default(); + let previous_digest = previous_digest_of(&active, entry.shard_id); + let result = read_shard_into( + store, + &active, + &entry, + &previous_digest, + genesis, + &scripts_here, + geometry, + &mut clients, + filters, + transport, + &mut charges, + limits, + target_anchor, + chain, + &mut failures, + ) + .await; + match result { + Ok(ShardRead::Done { matched }) => { + if !matched.is_empty() { + if !matched_shards.contains(&entry.shard_id) { + matched_shards.push(entry.shard_id); + } + for script in matched { + if !active_scripts.contains(&script) { + active_scripts.push(script); + } + } + } + position += 1; + } + Ok(ShardRead::Unproductive { count }) => { + unproductive += count; + position += 1; + } + Ok(ShardRead::Stopped(stopped)) => { + completion = stopped; + break 'passes; + } + Err(SyncError::StaleRevision { stale, .. }) => { + // The service withdrew this revision while the sync was + // reading it. Refresh the map, check it is the same set + // continued, roll this shard's provisional coverage back, + // and re-derive from whatever replaced it. + // + // Nothing read ahead survives a refresh: the rest of this + // pass is walked in sequence. + ahead::clear(transport, filters, &mut clients); + if refreshes >= MAX_MAP_REFRESHES + || (stale.map_sha256.is_some() + && stale.map_sha256.as_deref() == held_map_digest.as_deref()) + { + return Err(SyncError::StaleRevision { + shard_id: entry.shard_id, + stale, + refreshes, + }); + } + let (covered, _) = coverage_summary(store, &active, target_anchor.height)?; + let (fresh, digest) = + refresh_map(&active, covered, filters, &mut charges).await?; + check_target_hash(&fresh, target_anchor)?; + if fresh + .shards + .last() + .is_none_or(|s| s.end_height < target_anchor.height) + { + completion = Completion::Incomplete { + reason: IncompleteReason::PublicationBehind { + height: target_anchor.height, + }, + pending: store.pending()?.len(), + }; + active = Cow::Owned(fresh); + break 'passes; + } + refreshes += 1; + held_map_digest = Some(digest); + let Some(resume) = fresh + .shards + .iter() + .position(|candidate| candidate.shard_id == entry.shard_id) + else { + // The refreshed map no longer advertises this range. + active = Cow::Owned(fresh); + break; + }; + if fresh.shards[resume].manifest_digest == stale.revision { + return Err(SyncError::StaleRevision { + shard_id: entry.shard_id, + stale, + refreshes, + }); + } + check_resume(&fresh.shards[resume], &entry).map_err(SyncError::MapDiverged)?; + let resume_start = fresh.shards[resume].start_height; + let below = resume_start.saturating_sub(1); + store.rollback_above( + &accepted_at(chain, &active, below)?, + "revision withdrawn mid-sync", + )?; + if rolled_back_to.is_none_or(|held| held > below) { + rolled_back_to = Some(below); + } + if !replaced_revisions.contains(&stale.revision) { + replaced_revisions.push(stale.revision.clone()); + } + active = Cow::Owned(fresh); + // Re-plan against the new map from this shard on. + let mut rest: Vec = (resume..active.shards.len()) + .take_while(|i| active.shards[*i].start_height <= through) + .collect(); + index_iter.truncate(position); + index_iter.append(&mut rest); + for later in resume..active.shards.len() { + work.entry(later).or_insert_with(|| scripts_here.clone()); + } + } + Err(other) => return Err(other), + } + } + // Anything read ahead and not consumed belongs to this pass only. + ahead::clear(transport, filters, &mut clients); + + let added = scripts.on_activity(&active_scripts); + if added.is_empty() { + break; + } + scripts_added += store.add_scripts(&added)?; + } + + let (covered_through, settled_through) = + coverage_summary(store, &active, target_anchor.height)?; + if completion == Completion::Complete { + if covered_through < target_anchor.height { + // A failed entry leaves its script uncovered over its shard. + completion = Completion::Incomplete { + reason: if failures.entries.is_empty() { + IncompleteReason::DiscoveryUnbounded + } else { + IncompleteReason::ContentFailed + }, + pending: store.pending()?.len(), + }; + } else if !store.pending()?.is_empty() { + completion = Completion::Incomplete { + reason: IncompleteReason::PendingLimit, + pending: store.pending()?.len(), + }; + } else if !store.ledger()?.unresolved().is_empty() { + completion = Completion::Incomplete { + reason: IncompleteReason::UnresolvedSpends, + pending: 0, + }; + } else { + store.commit_anchor(target_anchor, settled_through, covered_through)?; + } + } + matched_shards.sort_unstable(); + let mut failed = failures.entries; + failed.sort_by(|a, b| (a.shard_id, &a.script).cmp(&(b.shard_id, &b.script))); + Ok(SyncReport { + ledger: store.ledger()?, + charges, + matched_shards, + unproductive_matches: unproductive, + covered_through, + settled_through, + provisional: provisional_of(store, &active)?, + map_refreshes: refreshes, + completion, + rolled_back_to, + replaced_revisions, + scripts_added, + commits: store.last_commit()? - first_commit, + failed_entries: failed, + }) +} + +/// Whether the map still vouches for a settled range the store holds. +/// +/// The range is looked up by the height it starts at, not by its shard id: a +/// re-cut renumbers shards, so the id it was read under may now name another +/// range. The map vouches for it when it still publishes that revision there, +/// or when one of its re-cuts declares exactly that revision superseded: the +/// same digest, shard id and start height, sealed, ending on the same block. +/// That keeps a wallet's history across the re-cut without re-reading it. +/// +/// The block a range ends on is its source anchor, the published endpoint, +/// when the store kept one, which every store since schema 2 does. A range +/// without one matches a declaration when it ends inside the declared range, +/// or at its end on the declared block. A range with an endpoint but cut +/// short of its shard's end by a target or a rollback is left to the chain +/// checks, as is anything the map's sealed shards do not cover. +fn vouches_for( + map: &ShardMap, + declared: &BTreeMap<&str, &transparent_filter::SupersededShard>, + range: &CoverageRange, +) -> bool { + if range + .source_anchor + .as_ref() + .is_some_and(|source| source.height != range.end_height) + { + return true; + } + let Some(entry) = map.shard_for_height(range.start_height) else { + return true; + }; + if !entry.sealed { + return true; + } + if entry.start_height == range.start_height + && entry.manifest_digest == range.revision_digest + && range.source_anchor.as_ref().is_none_or(|source| { + source.height == entry.end_height && source.hash == entry.terminal_block_hash + }) + { + return true; + } + declared + .get(range.revision_digest.as_str()) + .is_some_and(|declared| { + let ends_as_declared = match &range.source_anchor { + Some(source) => { + declared.end_height == source.height + && declared.terminal_block_hash == source.hash + } + // Without a recorded endpoint a range may have been cut short + // of its shard by a target or a rollback, as one with an + // endpoint is above: inside the declared range it matches, + // and at the declared end it must rest on the declared block. + None => { + range.end_height < declared.end_height + || (range.end_height == declared.end_height + && range.terminal_block_hash == declared.terminal_block_hash) + } + }; + declared.sealed + && declared.shard_id == range.shard_id + && declared.start_height == range.start_height + && ends_as_declared + }) +} + +/// What a settled range the map no longer vouches for means. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Rewrite { + /// The wallet's chain rejects the block the range rests on: a reorg, + /// rolled back like any other. + Reorg, + /// The wallet's chain accepts both the block the range rests on and the + /// one the shard now covering its start ends on: the publisher + /// contradicts itself on the chain the wallet follows. Retrying does not + /// change that while it serves this history. + Contradiction, + /// The chain cannot settle it yet; the height is the block it would need + /// to confirm. A publisher that followed a shallow reorg across a shard it + /// had just sealed, before the wallet's chain did, looks like this, and + /// becomes an ordinary reorg once the chain moves. + Unsettled(u64), +} + +/// Judges a settled range `vouches_for` refused, by what the wallet's chain +/// says about the block it rests on and about the block the map's shard now +/// covering its start ends on. +/// +/// Sealing is not delayed for finality, so a publisher can follow a reorg +/// through a just-sealed shard before the wallet's chain does. Only when the +/// chain accepts the replacement's end too, at or below the target, is the +/// change the publisher's alone. +fn judge_rewrite( + map: &ShardMap, + range: &CoverageRange, + target: &Anchor, + chain: &impl ChainView, +) -> Rewrite { + match chain.is_accepted(range.end_height, &range.terminal_block_hash) { + Acceptance::Rejected => Rewrite::Reorg, + Acceptance::Unknown => Rewrite::Unsettled(range.end_height), + Acceptance::Accepted => match map.shard_for_height(range.start_height) { + Some(entry) + if entry.sealed + && entry.end_height <= target.height + && chain.is_accepted(entry.end_height, &entry.terminal_block_hash) + == Acceptance::Accepted => + { + Rewrite::Contradiction + } + Some(entry) => Rewrite::Unsettled(entry.end_height), + // `vouches_for` refuses only ranges a sealed shard covers, so + // this and the unsealed case above are not reached; both would + // be the publisher ahead of or apart from the chain, not settled. + None => Rewrite::Unsettled(range.end_height), + }, + } +} + +/// What becomes of unfinished page work under a revision the map no longer +/// publishes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Unfinished { + /// Dropped, and what the revision saved stays. + Drop, + /// Dropped, and the store rolls back from this height. + Rewind(u64), +} + +/// Decides [`Unfinished`] for one item, from what the wallet itself stored. +/// +/// Every event an item saved lies at or below the target it was read for, +/// which the store keeps with it. So when a declared re-cut replaced the +/// sealed revision and the wallet's chain still accepts that target block, +/// what was saved is history on the chain the wallet follows, which a re-cut +/// does not change: it stays, the script's gap is read again under the shard +/// now covering it, and events are kept once per outpoint. The declaration +/// only says this is a re-cut; nothing it claims about heights or blocks is +/// relied on. +/// +/// Otherwise what the revision saved is rolled back from the lowest height it +/// saved anything at, or the declared start if that is lower; never from the +/// declared start alone, which the wallet has not checked. A revision that +/// saved nothing leaves nothing to roll back. +fn unfinished( + pending: &PendingPages, + declared: Option<&transparent_filter::SupersededShard>, + lowest_saved: Option, + chain: &impl ChainView, +) -> Unfinished { + let declared = declared.filter(|declared| declared.shard_id == pending.shard_id); + let target_held = pending.target_anchor.as_ref().is_some_and(|target| { + chain.is_accepted(target.height, &target.hash) == Acceptance::Accepted + }); + if declared.is_some_and(|declared| declared.sealed) && target_held { + return Unfinished::Drop; + } + match lowest_saved { + Some(lowest) => Unfinished::Rewind( + declared.map_or(lowest, |declared| declared.start_height.min(lowest)), + ), + None => Unfinished::Drop, + } +} + +/// Whether `script` has stored history inside `entry`'s range, up to the +/// target. +/// +/// By height rather than by the revision that recorded it: an event is kept +/// once per outpoint, so history first read under a revision a re-cut has +/// since replaced keeps that revision's name when the new shard is read. +fn holds_history( + events: &[StoredEvent], + script: &[u8], + entry: &transparent_filter::ShardMapEntry, + through: u64, +) -> bool { + let range = entry.start_height..=entry.end_height.min(through); + events + .iter() + .any(|stored| stored.script == script && range.contains(&event_height(&stored.event))) +} + +fn previous_digest_of(map: &ShardMap, shard_id: u64) -> String { + shard_id + .checked_sub(1) + .and_then(|before| map.shards.get(before as usize)) + .map(|entry| entry.manifest_digest.clone()) + .unwrap_or_default() +} + +/// Heights through which every script is covered from its required height: +/// provisional coverage included, then settled only. The minimum over +/// scripts, so one script's gap holds the whole wallet's figure down. +fn coverage_summary( + store: &S, + map: &ShardMap, + through: u64, +) -> Result<(u64, u64), SyncError> { + let floor = map.start_height.saturating_sub(1); + let mut covered = through; + let mut settled = through; + let entries = store.scripts()?; + if entries.is_empty() { + return Ok((floor, floor)); + } + for entry in entries { + let from = entry.required_from.max(map.start_height); + let ranges = store.coverage(&entry.script)?; + let all = uncovered(&ranges, from, through); + let mine = all + .first() + .map_or(through, |(start, _)| start.saturating_sub(1)); + covered = covered.min(mine); + let settled_ranges: Vec = ranges + .iter() + .filter(|range| range.kind == CoverageKind::Settled) + .cloned() + .collect(); + let settled_gaps = uncovered(&settled_ranges, from, through); + let mine_settled = settled_gaps + .first() + .map_or(through, |(start, _)| start.saturating_sub(1)); + settled = settled.min(mine_settled); + } + Ok((covered.max(floor), settled.max(floor))) +} + +fn provisional_of( + store: &S, + map: &ShardMap, +) -> Result, SyncError> { + let mut seen: BTreeMap<(u64, String), u64> = BTreeMap::new(); + for range in store.provisional()? { + let end = seen + .entry((range.shard_id, range.revision_digest.clone())) + .or_insert(range.end_height); + *end = (*end).max(range.end_height); + } + Ok(seen + .into_iter() + .map( + |((shard_id, manifest_digest), end_height)| ProvisionalCoverage { + shard_id, + // The digest identifies the revision; its number is the map's. + revision: map + .shards + .get(shard_id as usize) + .filter(|entry| entry.manifest_digest == manifest_digest) + .map(|entry| entry.revision) + .unwrap_or(0), + manifest_digest, + end_height, + }, + ) + .collect()) +} + +enum ShardRead { + /// Every script in scope was covered; `matched` names those with history. + Done { matched: Vec> }, + /// The filter matched but nothing was found; coverage advanced anyway. + Unproductive { count: u64 }, + /// A budget or refusal stopped the sync with the store consistent. + Stopped(Completion), +} + +/// Reads one shard for the scripts that need it, committing to the store. +#[allow(clippy::too_many_arguments)] +async fn read_shard_into( + store: &mut S, + map: &ShardMap, + entry: &transparent_filter::ShardMapEntry, + previous_digest: &str, + genesis: BlockHash, + scripts: &[Vec], + geometry: &ServiceGeometry, + clients: &mut Clients, + filters: &mut impl FilterSource, + transport: &mut impl ShardTransport, + charges: &mut ByteCharges, + limits: &WorkLimits, + target_anchor: &Anchor, + chain: &impl ChainView, + failures: &mut Failures, +) -> Result { + let endpoint = endpoint_of(entry, target_anchor); + match chain.is_accepted(endpoint.height, &endpoint.hash) { + Acceptance::Accepted => {} + Acceptance::Unknown => { + return Ok(ShardRead::Stopped(Completion::Incomplete { + reason: IncompleteReason::ChainUnknown { + height: endpoint.height, + }, + pending: store.pending()?.len(), + })) + } + Acceptance::Rejected => { + return Err(SyncError::Invalid( + "published coverage endpoint rejected by wallet chain".into(), + )) + } + } + // Ordinary sources download every filter in range. The explicit experiment + // may establish a parent negative; both paths use the same bounded commit. + let cached = store.filter(&entry.manifest_digest, &entry.filter_hash)?; + let (parent_negative, parent_cost) = if cached.is_none() { + filters + .parent_negative(map, entry.shard_id, scripts, store) + .await + .map_err(|e| SyncError::Transport(e.to_string()))? + } else { + (false, 0) + }; + charges.filter_bytes += parent_cost; + let matches = if parent_negative { + Vec::new() + } else { + let bytes = match cached { + Some(bytes) => bytes, + None => { + let (bytes, cost) = filters.filter(entry.shard_id).await.map_err(|error| { + SyncError::Transport(crate::transport::describe_error(error.as_ref())) + })?; + charges.filter_bytes += cost; + // Bind the bytes to the map before believing anything they say. + if transparent_filter::filter_hash(&bytes).to_display_hex() != entry.filter_hash { + // The public filter URL is shard-addressed: activation may + // have replaced it since this map was fetched. Never cache or + // match these bytes. The bounded stale-revision path requires + // a changed, validated map before retrying; an unchanged map + // still fails, including when the filter was simply corrupt. + return Err(SyncError::StaleRevision { + shard_id: entry.shard_id, + stale: StaleRevision { + shard_id: entry.shard_id, + revision: entry.manifest_digest.clone(), + map_sha256: None, + }, + refreshes: 0, + }); + } + store.put_filter( + &entry.manifest_digest, + &entry.filter_hash, + entry.sealed, + &bytes, + )?; + bytes + } + }; + charges.filters_checked += 1; + match_filter(map, entry, genesis, scripts, &bytes)? + }; + if matches.is_empty() { + // A genuinely empty range for every script here: coverage advances + // with no private request. + commit_bounded( + store, + target_anchor, + ShardCommit { + source_anchor: None, + shard_id: entry.shard_id, + revision_digest: entry.manifest_digest.clone(), + sealed: entry.sealed, + start_height: entry.start_height, + end_height: entry.end_height, + terminal_block_hash: entry.terminal_block_hash.clone(), + events: Vec::new(), + covered_scripts: scripts.to_vec(), + pending_upsert: Vec::new(), + pending_complete: Vec::new(), + cached_runs: Vec::new(), + failed_entries: Vec::new(), + }, + )?; + return Ok(ShardRead::Done { + matched: Vec::new(), + }); + } + // A geometry this build does not know is fatal the moment there is + // something to fetch, whatever the manifest would say about it. + if transparent_shard::layout::by_name(&entry.geometry).is_none() { + return Err(SyncError::UnknownGeometry(entry.geometry.clone())); + } + let matched_all: Vec> = matches.iter().map(|&i| scripts[i].clone()).collect(); + let unmatched: Vec> = scripts + .iter() + .filter(|script| !matched_all.contains(script)) + .cloned() + .collect(); + // An entry held out of this sync is not queried at all, and stays + // uncovered over this shard. + let mut matched_scripts = Vec::with_capacity(matched_all.len()); + for script in matched_all { + if !hold(store, &mut failures.entries, entry.shard_id, &script)? { + matched_scripts.push(script); + } + } + if matched_scripts.is_empty() { + if !unmatched.is_empty() { + commit_bounded( + store, + target_anchor, + ShardCommit { + source_anchor: None, + shard_id: entry.shard_id, + revision_digest: entry.manifest_digest.clone(), + sealed: entry.sealed, + start_height: entry.start_height, + end_height: entry.end_height, + terminal_block_hash: entry.terminal_block_hash.clone(), + covered_scripts: unmatched, + ..Default::default() + }, + )?; + } + return Ok(ShardRead::Done { + matched: Vec::new(), + }); + } + + let outcome = with_refusals( + store, + map, + entry, + previous_digest, + geometry, + clients, + transport, + charges, + limits, + async |store, prepared, transport, charges, manifest| { + // Validated when the manifest was verified; decoded again here + // rather than carried, since it is a few kilobytes. + let choice = directory_choice(manifest)?; + let salt = tag_salt_of(manifest)?; + retrieve_shard_into( + store, + entry, + manifest, + &matched_scripts, + &unmatched, + choice.as_ref(), + &salt, + prepared, + transport, + charges, + limits, + target_anchor, + chain, + failures, + ) + .await + }, + ) + .await?; + match outcome { + Some(stopped) => Ok(ShardRead::Stopped(stopped)), + None => { + // Which matched scripts actually held history is what the + // caller's gap-limit rule wants; the store knows. + let mut found = Vec::new(); + let mut unproductive = 0u64; + let events = store.events()?; + for script in &matched_scripts { + if failures + .entries + .iter() + .any(|held| held.shard_id == entry.shard_id && held.script == *script) + { + continue; + } + if holds_history(&events, script, entry, target_anchor.height) { + found.push(script.clone()); + } else { + unproductive += 1; + } + } + if found.is_empty() { + Ok(ShardRead::Unproductive { + count: unproductive, + }) + } else { + Ok(ShardRead::Done { matched: found }) + } + } + } +} + +/// The block a shard's coverage ends on for this sync: its own terminal +/// block, or the target when the shard runs past it. +fn endpoint_of(entry: &transparent_filter::ShardMapEntry, target_anchor: &Anchor) -> Anchor { + if entry.end_height > target_anchor.height { + target_anchor.clone() + } else { + Anchor { + height: entry.end_height, + hash: entry.terminal_block_hash.clone(), + } + } +} + +/// Which of `scripts` a shard's filter matches, by index. +fn match_filter( + map: &ShardMap, + entry: &transparent_filter::ShardMapEntry, + genesis: BlockHash, + scripts: &[Vec], + bytes: &[u8], +) -> Result, SyncError> { + let profile = transparent_filter::range_profile(&map.profile) + .ok_or_else(|| SyncError::UnknownProfile(map.profile.clone()))?; + let validated = + transparent_filter::validate_range_filter(bytes, FilterLimits::default(), profile)?; + let terminal = BlockHash::from_display_hex(&entry.terminal_block_hash)?; + let key = ShardKey::derive( + &map.profile, + genesis, + entry.shard_id, + entry.start_height, + entry.end_height, + terminal, + ); + let owned: Vec = scripts + .iter() + .map(|script| ScriptBytes::new(script.clone())) + .collect(); + Ok(transparent_filter::match_range_scripts( + &validated, key, &owned, + )?) +} + +/// Runs `work` against a shard, verifying the manifest first and lifting the +/// service's two refusals: a withdrawn revision propagates for the map +/// refresh. An overload returns incomplete so the host can schedule a later pass. +#[allow(clippy::too_many_arguments)] +async fn with_refusals( + store: &mut S, + map: &ShardMap, + entry: &transparent_filter::ShardMapEntry, + previous_digest: &str, + geometry: &ServiceGeometry, + clients: &mut Clients, + transport: &mut T, + charges: &mut ByteCharges, + limits: &WorkLimits, + mut work: F, +) -> Result, SyncError> +where + F: AsyncFnMut( + &mut S, + &mut GeometryClients, + &mut T, + &mut ByteCharges, + &ShardManifest, + ) -> Result, SyncError>, +{ + let _ = limits; + let mut manifest: Option = None; + loop { + let verified = match &manifest { + Some(verified) => verified, + None => match fetch_manifest(entry, previous_digest, map, transport, charges).await { + Ok(verified) => { + if verified.schema != geometry.schema { + return Err(SyncError::ManifestMismatch { + shard_id: entry.shard_id, + field: "schema", + }); + } + manifest.insert(verified) + } + Err(SyncError::Client(ClientError::Stale(stale))) => { + return Err(SyncError::StaleRevision { + shard_id: entry.shard_id, + stale, + refreshes: 0, + }) + } + Err(SyncError::Client(ClientError::Overloaded(_overloaded))) => { + return Ok(Some(Completion::Incomplete { + reason: IncompleteReason::Overloaded { + shard_id: entry.shard_id, + }, + pending: store.pending()?.len(), + })); + } + Err(other) => return Err(other), + }, + }; + let prepared = clients.prepare(&verified.geometry, &geometry.geometries)?; + match work(store, prepared, transport, charges, verified).await { + Ok(stopped) => return Ok(stopped), + Err(SyncError::Client(ClientError::Stale(stale))) => { + return Err(SyncError::StaleRevision { + shard_id: entry.shard_id, + stale, + refreshes: 0, + }) + } + Err(SyncError::Client(ClientError::Overloaded(_overloaded))) => { + // Nothing partial is lost: every commit so far stands, and + // the work resumes from the store on the next attempt. + return Ok(Some(Completion::Incomplete { + reason: IncompleteReason::Overloaded { + shard_id: entry.shard_id, + }, + pending: store.pending()?.len(), + })); + } + Err(SyncError::Client(ClientError::SetupMismatch { + revision, + table, + segment, + })) => { + // The setup held for that segment is not the one the service + // answers under. It is forgotten, here and in the store, so + // the next attempt fetches it again; this read still fails, + // as any failed request does, and counts against no entry. + let prepared = clients.prepare(&verified.geometry, &geometry.geometries)?; + match table { + Table::Directory => prepared.directory.forget(&revision, table, segment), + Table::Pages(_) => prepared.pages.forget(&revision, table, segment), + } + let set_digest = store + .set_identity()? + .map(|identity| identity.digest()) + .unwrap_or_default(); + store.drop_setup(&SetupKey { + set_digest, + revision_digest: revision.clone(), + table, + segment, + })?; + return Err(SyncError::Client(ClientError::SetupMismatch { + revision, + table, + segment, + })); + } + Err(other) => return Err(other), + } + } +} + +/// Whether a budget has been reached. +fn budget_stopped(charges: &ByteCharges, limits: &WorkLimits) -> Option { + if let Some(max) = limits.max_queries { + if charges.queries() >= max { + return Some(IncompleteReason::QueryBudget); + } + } + if let Some(max) = limits.max_private_bytes { + let private = charges.setup_bytes() + charges.query_upload() + charges.query_download(); + if private >= max { + return Some(IncompleteReason::ByteBudget); + } + } + None +} + +/// Fetches a matched shard's manifest and verifies it against the map. +async fn fetch_manifest( + entry: &transparent_filter::ShardMapEntry, + previous_digest: &str, + map: &ShardMap, + transport: &mut impl ShardTransport, + charges: &mut ByteCharges, +) -> Result { + let (raw, cost) = transport + .manifest(entry.shard_id, &entry.manifest_digest) + .await + .map_err(|error| SyncError::from(classify_transport(error)))?; + charges.add_manifest(cost); + verify_manifest(&raw, entry, previous_digest, map) +} + +/// Checks a manifest against the map entry that named it, the entry before +/// it, and this build's registry. +/// +/// The digest first: the bytes must hash to the digest the map names, or +/// nothing else in them is worth reading. Then every field a retrieval relies +/// on. The parent link ties the manifest to the map's previous entry, so a +/// manifest that is individually well formed but belongs to a different +/// sequence of shards is refused too. Segment shapes are checked against the +/// registry entry the manifest names, which is what makes the geometry a +/// property of the verified manifest rather than of the map alone. +/// +/// None of this is an independence proof. The map, the manifest and the +/// tables come from one publisher; agreement among them rules out corruption, +/// stale data and mixed publications, not omission. +fn verify_manifest( + raw: &[u8], + entry: &transparent_filter::ShardMapEntry, + previous_digest: &str, + map: &ShardMap, +) -> Result { + let shard_id = entry.shard_id; + let mismatch = |field: &'static str| SyncError::ManifestMismatch { shard_id, field }; + let manifest: ShardManifest = serde_json::from_slice(raw) + .map_err(|error| SyncError::Invalid(format!("shard {shard_id} manifest: {error}")))?; + if manifest.directory_segments.len() > 64 + || manifest.page_children.len() > 32 + || manifest + .page_children + .iter() + .any(|c| c.page_segments.len() > 64) + { + return Err(mismatch("resource_bounds")); + } + if manifest.digest() != entry.manifest_digest { + return Err(mismatch("digest")); + } + if !transparent_shard::manifest::supported_schema(&manifest.schema) { + return Err(SyncError::Schema { + served: manifest.schema.clone(), + expected: transparent_shard::SCHEMA, + }); + } + if manifest.layout != transparent_shard::ManifestLayout::current() { + return Err(mismatch("layout")); + } + if manifest.tag_salt_counter > transparent_shard::tag::MAX_TAG_SALT_COUNTER { + return Err(mismatch("tag_salt_counter")); + } + if manifest.network != map.network { + return Err(mismatch("network")); + } + if manifest.genesis_hash != map.genesis_hash { + return Err(mismatch("genesis_hash")); + } + if manifest.profile != map.profile { + return Err(mismatch("profile")); + } + if manifest.shard_id != entry.shard_id { + return Err(mismatch("shard_id")); + } + if manifest.geometry != entry.geometry { + return Err(mismatch("geometry")); + } + if manifest.start_height != entry.start_height || manifest.end_height != entry.end_height { + return Err(mismatch("range")); + } + if manifest.parent_block_hash != entry.parent_block_hash { + return Err(mismatch("parent_block_hash")); + } + if manifest.terminal_block_hash != entry.terminal_block_hash { + return Err(mismatch("terminal_block_hash")); + } + if manifest.filter_hash != entry.filter_hash { + return Err(mismatch("filter_hash")); + } + if manifest.sealed != entry.sealed { + return Err(mismatch("sealed")); + } + if manifest.revision != entry.revision { + return Err(mismatch("revision")); + } + if manifest.parent_manifest_digest != previous_digest { + return Err(mismatch("parent_manifest_digest")); + } + // The map's `page_segments` counts the directory shard's page shards. + if manifest.directory_segments.len() as u32 != entry.directory_segments + || manifest.page_children.len() as u32 != entry.page_segments + { + return Err(mismatch("segment_count")); + } + let geometry = transparent_shard::layout::by_name(&manifest.geometry) + .ok_or_else(|| SyncError::UnknownGeometry(manifest.geometry.clone()))?; + for segment in &manifest.directory_segments { + if segment.rows != geometry.directory_rows + || segment.row_bytes != geometry.directory_row_bytes as u32 + { + return Err(mismatch("directory_segment_shape")); + } + } + // Every page shard: contiguous, in order, sealed but for an open last one + // of a tail, ending on the shard's terminal, segments of the registry's + // shape. A walk relies on each of these. + manifest + .check_children(geometry.page_rows, geometry.page_row_bytes as u32) + .map_err(|_| mismatch("page_children"))?; + for child in &manifest.page_children { + BlockHash::from_display_hex(&child.terminal_block_hash) + .map_err(|_| mismatch("page_children"))?; + } + if let Some(seal) = map.seal.get(&manifest.geometry) { + if manifest.seal.scripts_target != seal.max_scripts + || manifest.seal.page_rows_target != seal.max_page_rows + { + return Err(mismatch("seal")); + } + } + directory_choice(&manifest)?; + Ok(manifest) +} + +/// The manifest's directory choice table, if it publishes one. +/// +/// A table that does not decode, or that indexes a different number of +/// scripts than the shard places, stops the sync. Falling back to two queries +/// would be safe for this shard, but it would hide a malformed publication +/// behind a working one. +fn directory_choice( + manifest: &ShardManifest, +) -> Result, SyncError> { + manifest.directory_choice().map_err(|error| { + SyncError::Invalid(format!("shard {} manifest: {error}", manifest.shard_id)) + }) +} + +/// How long to wait before re-asking an overloaded service. +/// +/// The service's own figure is preferred, because it knows why it refused, but +/// it is capped: a delay a wallet cannot sanity-check is a delay a +/// misconfigured or hostile service could use to park a sync inside one call. +/// Either way the wait is jittered by up to half again, so wallets refused +/// together do not return together. + +/// Refetches the map after a withdrawn revision, and checks it continues the +/// one in hand. +/// +/// Returns the map and the digest of the bytes it came from, so a later refusal +/// naming that same digest can be recognised as unresolvable rather than +/// spending another refresh on it. +async fn refresh_map( + started_from: &ShardMap, + covered_through: u64, + filters: &mut impl FilterSource, + charges: &mut ByteCharges, +) -> Result<(ShardMap, String), SyncError> { + let (bytes, cost) = filters + .shard_map() + .await + .map_err(|error| SyncError::Transport(crate::transport::describe_error(error.as_ref())))?; + charges.map_bytes += cost; + let fresh: ShardMap = serde_json::from_slice(&bytes).map_err(|error| { + SyncError::Invalid(format!("refreshed shard map is malformed: {error}")) + })?; + fresh.check_shape().map_err(|error| { + SyncError::Invalid(format!("refreshed shard map is malformed: {error}")) + })?; + check_continuation(started_from, &fresh, covered_through).map_err(SyncError::MapDiverged)?; + let digest = hex::encode(::digest(&bytes)); + Ok((fresh, digest)) +} + +/// Checks a refreshed map is the same shard set, continued. +/// +/// A map fetched mid-sync is not independently trusted. The caller validated +/// the map this sync started from against the wallet's accepted chain, and a +/// refresh inherits that validation only over the range the two agree on. So +/// the set's identity must be unchanged, and every shard whose range is already +/// covered must be the *same entry* — sealed content is immutable, so a +/// disagreement there means a different set rather than a longer one. +/// +/// Everything this function rules out is what makes the recovery sound: once it +/// passes, the shard the sync resumes at begins exactly where coverage stopped, +/// so no range can be skipped and none can need retracting. +fn check_continuation( + started_from: &ShardMap, + fresh: &ShardMap, + covered_through: u64, +) -> Result<(), String> { + if fresh.genesis_hash != started_from.genesis_hash { + return Err("it names a different genesis block".into()); + } + if fresh.network != started_from.network { + return Err("it names a different network".into()); + } + if fresh.profile != started_from.profile { + return Err("it names a different filter profile".into()); + } + if fresh.range_envelope_version != started_from.range_envelope_version { + return Err("it names a different range envelope version".into()); + } + if fresh.start_height != started_from.start_height { + return Err("its coverage begins at a different height".into()); + } + // A re-cut redraws the boundaries this walk was planned over and gives + // shard ids to other ranges, even where it lies above everything covered + // so far. This sync ends; the next starts from the re-cut map, whose + // declaration lets it keep what the store holds. + if fresh.recut_epoch() != started_from.recut_epoch() { + return Err(format!( + "it is re-cut epoch {}, this sync started from epoch {}", + fresh.recut_epoch(), + started_from.recut_epoch() + )); + } + // New geometries may appear as a set grows into another tier; the seal + // parameters of a geometry already in use may not change, because they are + // what the boundaries were derived from. + for (name, seal) in &started_from.seal { + if let Some(refreshed) = fresh.seal.get(name) { + if refreshed != seal { + return Err(format!("it reseals geometry {name}")); + } + } + } + for (index, before) in started_from.shards.iter().enumerate() { + if before.end_height > covered_through { + break; + } + match fresh.shards.get(index) { + Some(after) if after == before => {} + Some(_) => { + return Err(format!( + "shard {} covers height {} but is not the shard this sync read", + before.shard_id, before.end_height + )) + } + None => { + return Err(format!( + "it withdraws shard {}, whose range this sync already covered", + before.shard_id + )) + } + } + } + // A map that ends below what is already covered withdraws history the + // wallet has read, which is the same disagreement as a re-cut shard. + if let Some(last) = fresh.shards.last() { + if last.end_height < covered_through { + return Err(format!( + "its coverage ends at {}, below the {covered_through} this sync already read", + last.end_height + )); + } + } else if covered_through >= started_from.start_height { + return Err("it publishes no shards at all".into()); + } + Ok(()) +} + +/// Checks the shard the walk resumes at after a refresh. +/// +/// The walk resumes by shard id, which is sound only while the id still names +/// a range starting where the withdrawn one did. `check_continuation` already +/// refuses a refreshed map that changes covered history or its re-cut epoch, +/// so with a well-formed map this holds; it is checked again here because +/// resuming anywhere else would skip or repeat heights. +fn check_resume( + resumed: &transparent_filter::ShardMapEntry, + withdrawn: &transparent_filter::ShardMapEntry, +) -> Result<(), String> { + if resumed.start_height != withdrawn.start_height { + return Err(format!( + "it moves shard {} from height {} to {}", + withdrawn.shard_id, withdrawn.start_height, resumed.start_height + )); + } + Ok(()) +} + +/// Salt the wallet derives from a verified manifest. Records are accepted +/// only under this salt. +fn tag_salt_of(manifest: &ShardManifest) -> Result<[u8; 32], SyncError> { + let terminal = BlockHash::from_display_hex(&manifest.terminal_block_hash).map_err(|error| { + SyncError::Invalid(format!( + "shard {} terminal block hash: {error}", + manifest.shard_id + )) + })?; + Ok(transparent_shard::tag_salt( + manifest.shard_id, + &terminal, + manifest.tag_salt_counter, + )) +} + +/// The directory phase for one shard: the candidate rows for every matched +/// script, inline events committed, chain walks recorded; then the walks. +/// +/// A script whose replies fail on content fails its entry alone: it gets no +/// coverage and no walk, as an entry with inline history or a false-positive +/// match sends no pages either, and the shard's one commit still lands for +/// every other script, counting the failure. Every directory query planned for +/// a script is sent before any of its replies is decoded, so the queries sent +/// are the same whatever came back, and in the order a lookahead stashed them. +/// A retry after an overload sends them again for every script without a +/// saved walk, failed ones included, so it too is the honest retry. +#[allow(clippy::too_many_arguments)] +async fn retrieve_shard_into( + store: &mut S, + entry: &transparent_filter::ShardMapEntry, + manifest: &ShardManifest, + matched: &[Vec], + unmatched: &[Vec], + choice: Option<&transparent_shard::ChoiceTable>, + salt: &[u8; 32], + clients: &mut GeometryClients, + transport: &mut T, + charges: &mut ByteCharges, + limits: &WorkLimits, + target_anchor: &Anchor, + chain: &impl ChainView, + failures: &mut Failures, +) -> Result, SyncError> { + let shard_id = entry.shard_id; + let geometry = transparent_shard::layout::by_name(&entry.geometry) + .ok_or_else(|| SyncError::UnknownGeometry(entry.geometry.clone()))?; + let revision = entry.manifest_digest.as_str(); + // An overload can occur after the directory commit but before the walks + // finish. with_refusals calls this function again on retry. Resume those + // durable walks first: rediscovering their directories would enqueue + // duplicate work and leave the original walks pending after the retry. + let pending: Vec<_> = store + .pending()? + .into_iter() + .filter(|p| p.shard_id == shard_id && p.revision_digest == revision) + .collect(); + let mut resumed = Vec::with_capacity(pending.len()); + for item in &pending { + if !held(store, &failures.entries, shard_id, &item.script)? + || failures.owes(shard_id, &item.script).is_some() + { + resumed.push(item.clone()); + } + } + if !resumed.is_empty() { + if let Some(stopped) = finish_pages( + store, + entry, + manifest, + &resumed, + clients, + transport, + charges, + limits, + target_anchor, + chain, + failures, + ) + .await? + { + return Ok(Some(stopped)); + } + } + // A retry after an overload asks again for every matched script without + // a saved walk, as an honest retry does for inline entries and false + // positives. An entry that already failed in this sync is asked again + // too, so the queries do not show which entries failed, but its replies + // are not read: it is not counted twice, covered, or walked. + let mut unowed = Vec::with_capacity(matched.len()); + for script in matched { + if !pending.iter().any(|p| &p.script == script) { + let failed = held(store, &failures.entries, shard_id, script)?; + unowed.push((script, failed)); + } + } + let matched = unowed; + if matched.is_empty() { + // The earlier directory commit also covered all unmatched scripts. + return Ok(None); + } + let set_digest = store + .set_identity()? + .map(|identity| identity.digest()) + .unwrap_or_default(); + open_table( + store, + &set_digest, + shard_id, + revision, + Table::Directory, + entry.directory_segments, + &mut clients.directory, + transport, + charges, + None, + ) + .await?; + + if let Some(reason) = budget_stopped(charges, limits) { + return Ok(Some(Completion::Incomplete { + reason, + pending: store.pending()?.len(), + })); + } + + let mut events: Vec = Vec::new(); + let mut covered: Vec> = unmatched.to_vec(); + let mut pending_upsert: Vec = Vec::new(); + let mut failed_here: Vec> = Vec::new(); + + for (script, failed) in matched { + // With a published choice table, exactly the one candidate row it + // names is queried. Without one, both candidate rows are. Either way + // the query count per matched script is fixed for the shard: querying + // the first and stopping on a hit would make it depend on where the + // script landed, which is a function of the script. Candidates are + // taken over the shard's whole logical row space; each names one row + // within a segment, and every segment answers it. + let wanted = transparent_shard::script_tag(salt, script); + let rows = geometry.directory_rows * entry.directory_segments as u64; + let candidates = candidate_rows(shard_id, script, rows); + let queried: &[u64] = match choice { + Some(table) => std::slice::from_ref(&candidates[table.choice(shard_id, script)]), + None => &candidates, + }; + let mut replies = Vec::with_capacity(queried.len()); + for &row in queried { + let (_, within) = transparent_shard::layout::split_row(row, geometry.directory_rows); + let directory_rows = clients.directory.rows(); + let answer = clients + .directory + .fetch_row( + transport, + shard_id, + revision, + Table::Directory, + entry.directory_segments, + directory_rows, + within as usize, + charges, + ) + .await; + replies.push((row, answer?)); + } + if failed { + continue; + } + let found = match entry_in(entry, manifest, geometry, &wanted, &replies) { + Ok(found) => found, + Err(SyncError::Content(_)) => { + failed_here.push(script.clone()); + continue; + } + Err(other) => return Err(other), + }; + match found { + // A filter match with no directory entry is either a false positive + // or a script outside the private tables' coverage. Both are + // wasted work rather than errors; coverage still advances. + None => covered.push(script.clone()), + Some(found) => { + for event in &found.inline { + events.push(StoredEvent { + script: script.clone(), + event: *event, + shard_id, + revision_digest: revision.to_string(), + }); + } + match found.head() { + Some((child, row)) => pending_upsert.push(PendingPages { + id: None, + shard_id, + revision_digest: revision.to_string(), + script: script.clone(), + cursor: RunCursor { + child, + row, + older_runs: None, + }, + fragments: Vec::new(), + carried: found.inline.clone(), + attempts: 0, + target_anchor: Some(target_anchor.clone()), + }), + None => covered.push(script.clone()), + } + } + } + } + + // The directory phase is one commit: inline events, coverage for every + // script that needs no pages, and the walk owed for the rest. The walks + // that follow resume from the store even if nothing else of this call + // survives. + let pending_ids_before: std::collections::BTreeSet = + store.pending()?.iter().filter_map(|p| p.id).collect(); + commit_bounded( + store, + target_anchor, + ShardCommit { + source_anchor: None, + shard_id, + revision_digest: revision.to_string(), + sealed: entry.sealed, + start_height: entry.start_height, + end_height: entry.end_height, + terminal_block_hash: entry.terminal_block_hash.clone(), + events, + covered_scripts: covered, + pending_upsert, + pending_complete: Vec::new(), + cached_runs: Vec::new(), + failed_entries: failed_here.clone(), + }, + )?; + for script in &failed_here { + record_failure(store, &mut failures.entries, shard_id, script)?; + } + let owed: Vec = store + .pending()? + .into_iter() + .filter(|p| { + p.shard_id == shard_id + && p.revision_digest == revision + && p.id.is_some_and(|id| !pending_ids_before.contains(&id)) + }) + .collect(); + if owed.is_empty() { + return Ok(None); + } + finish_pages( + store, + entry, + manifest, + &owed, + clients, + transport, + charges, + limits, + target_anchor, + chain, + failures, + ) + .await +} + +/// The entry tagged `wanted` among the replies to one script's candidate +/// rows, if there is one, checked as far as it can be before a page is read: +/// every reply decodes, the tag is in at most one row, the inline events lie +/// in the shard, and the run it points to is a row a page query can select. +/// Any of these failing is a content failure of the entry. +fn entry_in( + entry: &transparent_filter::ShardMapEntry, + manifest: &ShardManifest, + geometry: &transparent_shard::layout::Geometry, + wanted: &[u8], + replies: &[(u64, Reply)], +) -> Result, SyncError> { + let shard_id = entry.shard_id; + let children = manifest.page_children.len() as u32; + let mut found: Option<(u64, DirectoryEntry)> = None; + for (row, reply) in replies { + for raw in reply { + let decoded = transparent_shard::records::decode_directory_row(raw, children) + .map_err(|error| SyncError::Content(format!("shard {shard_id}: {error}")))?; + for candidate in decoded { + // The row is selected by a hash of the raw script. The tag + // derived from this manifest's salt is what settles a hit. + // A tag from any other salt does not match. + if candidate.tag[..] != *wanted { + continue; + } + match &found { + // The two candidate hashes can name the same row. The row + // is still fetched twice, so the query count does not + // depend on the script, and the second sighting is the + // same entry, not a second. + Some((seen_row, _)) if seen_row == row => {} + Some(_) => { + return Err(SyncError::Content(format!( + "shard {shard_id} holds a script tag twice" + ))); + } + None => found = Some((*row, candidate)), + } + } + } + } + let Some((_, found)) = found else { + return Ok(None); + }; + if found.inline.iter().any(|event| { + let height = event_height(event); + height < entry.start_height || height > entry.end_height + }) { + return Err(SyncError::Content(format!( + "shard {shard_id} holds an inline event outside its range" + ))); + } + if let Some((child, row)) = found.head() { + locate(manifest, geometry, shard_id, child, u64::from(row))?; + } + Ok(Some(found)) +} + +/// Walks the chains still owed for one directory shard revision, privately. +/// +/// Every query of `pages-` selects among that page shard's first +/// [`PageChild::query_rows`]. See [`walk_chains`] for the walk itself. +#[allow(clippy::too_many_arguments)] +async fn finish_pages( + store: &mut S, + entry: &transparent_filter::ShardMapEntry, + manifest: &ShardManifest, + owed: &[PendingPages], + clients: &mut GeometryClients, + transport: &mut T, + charges: &mut ByteCharges, + limits: &WorkLimits, + target_anchor: &Anchor, + chain: &impl ChainView, + failures: &mut Failures, +) -> Result, SyncError> { + if entry.end_height < target_anchor.height { + match chain.is_accepted(entry.end_height, &entry.terminal_block_hash) { + Acceptance::Accepted => {} + Acceptance::Unknown => { + return Ok(Some(Completion::Incomplete { + reason: IncompleteReason::ChainUnknown { + height: entry.end_height, + }, + pending: store.pending()?.len(), + })) + } + Acceptance::Rejected => { + return Err(SyncError::Invalid( + "pending shard endpoint rejected by wallet chain".into(), + )) + } + } + } + let mut rows = PirRows { + entry, + manifest, + client: &mut clients.pages, + transport, + charges, + limits, + }; + walk_chains( + store, + entry, + manifest, + owed, + &mut rows, + target_anchor, + failures, + ) + .await +} + +/// One page row a walk asks for: page shard `child`, and a row over its +/// logical row space, which is its segments concatenated. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct RowRequest { + child: u32, + row: u64, +} + +/// One private reply: the row from every table segment. +/// +/// A reply that does not verify as a whole (its length, its revision binding, +/// the parameter epoch it was made under) fails its request, not an entry: +/// the service produces those without knowing which row the query selects, +/// so they cannot be aimed at one. Only the rows' own content, decoded and +/// checked, can fail an entry. +type Reply = Vec>; + +/// Where a walk's page rows come from: privately, or in tests, directly. +trait RowSource { + /// Why the walk must stop before another round, if it must. + fn stopped(&self) -> Option; + /// Replies for a prefix of the requests, in order. A budget may stop + /// dispatch between requests; each reply contains every table segment. + async fn fetch( + &mut self, + store: &mut S, + requests: &[RowRequest], + ) -> Result, SyncError>; + /// A query whose reply is not read: row 0 of page shard `child`. Every + /// query to a table selects among the same rows, so it is the length of + /// any other query to that table, and the service cannot tell it apart. + fn dummy(&self, child: u32) -> RowRequest { + RowRequest { child, row: 0 } + } +} + +/// Page rows by private query against one directory shard revision. +struct PirRows<'a, T: ShardTransport> { + entry: &'a transparent_filter::ShardMapEntry, + manifest: &'a ShardManifest, + client: &'a mut TableClient, + transport: &'a mut T, + charges: &'a mut ByteCharges, + limits: &'a WorkLimits, +} + +impl RowSource for PirRows<'_, T> { + fn stopped(&self) -> Option { + budget_stopped(self.charges, self.limits) + } + + /// Sends a round's queries together when the transport overlaps + /// requests, one at a time otherwise. Which queries are sent is the + /// walk's alone; a batch changes only when. + async fn fetch( + &mut self, + store: &mut S, + requests: &[RowRequest], + ) -> Result, SyncError> { + let shard_id = self.entry.shard_id; + let revision = self.entry.manifest_digest.as_str(); + let geometry = transparent_shard::layout::by_name(&self.entry.geometry) + .ok_or_else(|| SyncError::UnknownGeometry(self.entry.geometry.clone()))?; + let set_digest = store + .set_identity()? + .map(|identity| identity.digest()) + .unwrap_or_default(); + if self.limits.max_private_bytes.is_none() && self.transport.concurrency() > 1 { + let remaining = self + .limits + .max_queries + .map(|limit| limit.saturating_sub(self.charges.queries()) as usize) + .unwrap_or(requests.len()); + let requests = &requests[..requests.len().min(remaining)]; + // Each request: its table, segment count, query rows and row within. + let mut planned = Vec::with_capacity(requests.len()); + for request in requests { + let child = self + .manifest + .child(request.child) + .ok_or_else(|| SyncError::Invalid("a walk left the page shards".into()))?; + let table = Table::Pages(request.child); + let segments = child.page_segments.len() as u32; + open_table( + store, + &set_digest, + shard_id, + revision, + table, + segments, + self.client, + self.transport, + self.charges, + None, + ) + .await?; + let (_, within) = + transparent_shard::layout::split_row(request.row, geometry.page_rows); + planned.push(( + table, + segments, + child.query_rows() as usize, + within as usize, + )); + } + let mut answers = Vec::with_capacity(planned.len()); + if self.transport.concurrency() <= 1 || planned.len() <= 1 { + for (table, segments, query_rows, within) in planned { + answers.push( + self.client + .fetch_row( + self.transport, + shard_id, + revision, + table, + segments, + query_rows, + within, + self.charges, + ) + .await?, + ); + } + return Ok(answers); + } + let mut queries = Vec::with_capacity(planned.len()); + for (table, _, query_rows, within) in &planned { + queries.push( + self.client + .prepare_prefix(revision, *table, *query_rows, *within)?, + ); + } + let batch: Vec> = planned + .iter() + .zip(&queries) + .map(|((table, ..), query)| ShardRequest::Query { + shard_id, + revision, + table: *table, + body: &query.body, + }) + .collect(); + let replies = self.transport.batch(&batch).await; + drop(batch); + for (((table, segments, query_rows, within), query), reply) in + planned.into_iter().zip(queries).zip(replies) + { + match reply { + Some(Ok((response, _))) => { + self.charges.add_query( + table, + query.body.len() as u64, + response.len() as u64, + ); + answers.push( + self.client + .decode(revision, table, segments, query, &response)?, + ); + } + Some(Err(error)) => return Err(SyncError::Client(classify_transport(error))), + None => answers.push( + self.client + .fetch_row( + self.transport, + shard_id, + revision, + table, + segments, + query_rows, + within, + self.charges, + ) + .await?, + ), + } + } + return Ok(answers); + } + let mut answers = Vec::with_capacity(requests.len()); + // A finite byte allowance is checked after each atomic response. Opening + // all setups or dispatching a whole batch first would exceed that contract. + for request in requests { + if self.stopped().is_some() { + break; + } + let child = self + .manifest + .child(request.child) + .ok_or_else(|| SyncError::Invalid("a walk left the page shards".into()))?; + let table = Table::Pages(request.child); + let segments = child.page_segments.len() as u32; + if !open_table( + store, + &set_digest, + shard_id, + revision, + table, + segments, + self.client, + self.transport, + self.charges, + Some(self.limits), + ) + .await? + { + break; + } + if self.stopped().is_some() { + break; + } + let (_, within) = transparent_shard::layout::split_row(request.row, geometry.page_rows); + answers.push( + self.client + .fetch_row( + self.transport, + shard_id, + revision, + table, + segments, + child.query_rows() as usize, + within as usize, + self.charges, + ) + .await?, + ); + } + Ok(answers) + } +} + +/// The history cached below a run, and the page shard of the cached run it +/// starts from. +type CachedBelow = (u8, Vec); + +/// One script's chain walk in progress. +struct Walk { + item: PendingPages, + /// The first fragment of the run at the cursor, once read. + head: Option, + /// Runs read and committed in this call, newest first, with their page + /// shard: what the run cache is filled from once the walk ends. + walked: Vec<(u8, Vec)>, + done: bool, + /// After a content failure: the page shard and number of queries the + /// walk had committed to and not sent, sent as dummies in the next round, + /// after which the walk ends. + dummies: Option<(u8, u32)>, +} + +/// A walk's failure, with the queries valid replies had already committed it +/// to: `dummies` of them, to page shard `child`. +struct Failure { + error: SyncError, + child: u8, + dummies: u32, +} + +impl From for Failure { + fn from(error: SyncError) -> Self { + Self { + error, + child: 0, + dummies: 0, + } + } +} + +/// Walks the chains of runs `owed` describes, all scripts' rounds together, +/// newest run first, committing each run once it is read and checked. +/// +/// A round reads each run's first fragment or a prefix of its remaining +/// fragments. Checked fragments are saved between passes, so a one-query +/// budget can advance even a long run. A complete run is committed before +/// checking the next request's budget. A walk reaching a cached sealed run +/// stops there and takes that history. +/// +/// Refused, as content failures: a link to the same or a later page shard; +/// an `older_runs` that does not drop by exactly one per hop, which is how a +/// chain cut short or extended shows; fragments out of ordinal, count or +/// greedy boundary; events not strictly increasing from the oldest run into +/// the inline events; and an event in a page shard other than the one open +/// when it was pushed out — the event two places after it must lie in that +/// page shard's range. +/// +/// A content failure fails its walk alone; the other walks finish. The failed +/// walk keeps its pending item, so a later sync retries it, and the failure is +/// counted against the entry. It still sends what valid replies had already +/// committed it to, the fragments of its current run not yet asked for, as +/// dummy queries to that run's page shard, and then nothing. A run whose link +/// names a page shard the manifest does not declare, or a row outside that +/// table, sends one dummy to its own page shard in the link's place, unless +/// the honest walk would not query a head after this run. Nothing that +/// depends on the bad reply is sent. +async fn walk_chains( + store: &mut S, + entry: &transparent_filter::ShardMapEntry, + manifest: &ShardManifest, + owed: &[PendingPages], + rows: &mut R, + target_anchor: &Anchor, + failures: &mut Failures, +) -> Result, SyncError> { + let geometry = transparent_shard::layout::by_name(&entry.geometry) + .ok_or_else(|| SyncError::UnknownGeometry(entry.geometry.clone()))?; + let identities: Vec = manifest.child_digests().iter().map(hex::encode).collect(); + let salts = manifest + .page_children + .iter() + .map(|child| { + let terminal = BlockHash::from_display_hex(&child.terminal_block_hash)?; + Ok(transparent_shard::page_tag_salt( + manifest.shard_id, + child.index, + child.end_height, + &terminal, + child.tag_salt_counter, + )) + }) + .collect::, SyncError>>()?; + let walker = Walker { + entry, + manifest, + geometry, + identities: &identities, + salts: &salts, + target_anchor, + }; + let mut walks: Vec = Vec::with_capacity(owed.len()); + for item in owed { + let mut walk = Walk { + item: item.clone(), + head: None, + walked: Vec::new(), + done: false, + dummies: None, + }; + if let Some(owed) = failures.owes(entry.shard_id, &item.script) { + // Failed earlier in this sync, before a retry: it still sends + // what it owed. + walk.dummies = Some(owed); + } else if let Some(raw) = item.fragments.first() { + let head = walker + .fragment( + &item.script, + item.cursor.child, + std::slice::from_ref(raw), + 0, + None, + ) + .map_err(Failure::from) + .and_then(|head| { + walker.check_located( + store, + &item.script, + &head, + &item.cursor, + item.fragments.len(), + )?; + Ok(head) + }); + match head { + Ok(head) => walk.head = Some(head), + Err(failure) => walker.fail(store, &mut walk, failures, failure)?, + } + } + walks.push(walk); + } + loop { + // A walk at a run the store holds history for stops there. + for walk in walks + .iter_mut() + .filter(|w| !w.done && w.dummies.is_none() && w.head.is_none()) + { + let child = walk.item.cursor.child; + let stopped = match walker.cached(store, &walk.item.script, child) { + Ok(Some(older)) => walker.finish(store, walk, Vec::new(), Some((child, older))), + Ok(None) => Ok(()), + Err(error) => Err(error), + }; + if let Err(error) = stopped { + walker.fail(store, walk, failures, error.into())?; + } + } + if walks.iter().all(|walk| walk.done) { + return Ok(None); + } + if rows.stopped().is_some() { + for walk in walks.iter_mut().filter(|w| !w.done && w.dummies.is_none()) { + if walk + .head + .as_ref() + .is_some_and(|head| walk.item.fragments.len() == head.fragment_count as usize) + { + if let Err(failure) = walker.complete(store, walk) { + walker.fail(store, walk, failures, failure)?; + } + } + } + if walks.iter().all(|walk| walk.done) { + return Ok(None); + } + } + if let Some(reason) = rows.stopped() { + return Ok(Some(Completion::Incomplete { + reason, + pending: store.pending()?.len(), + })); + } + + // This round's rows, every walk's together. + let mut requests: Vec = Vec::new(); + let mut spans: Vec<(usize, usize, Option)> = Vec::new(); + for walk in walks.iter_mut() { + let start = requests.len(); + let mut older = None; + if walk.dummies.is_none() && !walk.done { + match walker.plan(store, walk) { + Ok((planned, below)) => { + requests.extend(planned); + older = below; + } + Err(error) => walker.fail(store, walk, failures, error.into())?, + } + } + if let Some((child, count)) = walk.dummies { + requests.extend((0..count).map(|_| rows.dummy(u32::from(child)))); + } + spans.push((start, requests.len(), older)); + } + let answers = rows.fetch(store, &requests).await?; + if answers.len() > requests.len() + || answers.is_empty() && !requests.is_empty() && rows.stopped().is_none() + { + return Err(SyncError::Invalid("a round lost an answer".into())); + } + for (walk, (start, end, older)) in walks.iter_mut().zip(spans) { + if walk.done { + continue; + } + // A failed walk's dummies were this round's; their replies are + // not read, and the walk ends. + if walk.dummies.take().is_some() { + failures.settle(entry.shard_id, &walk.item.script); + walk.done = true; + continue; + } + if start >= answers.len() && start != end { + continue; + } + let available = if start == end { + &[][..] + } else { + &answers[start..end.min(answers.len())] + }; + if let Err(failure) = walker.advance(store, walk, available, older) { + walker.fail(store, walk, failures, failure)?; + } + } + } +} + +/// What a walk checks against, for one directory shard revision. +struct Walker<'a> { + entry: &'a transparent_filter::ShardMapEntry, + manifest: &'a ShardManifest, + geometry: &'static transparent_shard::layout::Geometry, + /// Each page shard's chained identity, in hex: the run cache's key. + identities: &'a [String], + /// Each page shard's tag salt. + salts: &'a [[u8; 32]], + target_anchor: &'a Anchor, +} + +/// Page shard `child` of a directory shard, which its manifest must declare. +fn page_child(manifest: &ShardManifest, shard_id: u64, child: u8) -> Result<&PageChild, SyncError> { + manifest.child(u32::from(child)).ok_or_else(|| { + SyncError::Content(format!( + "shard {shard_id} names page shard {child}, which it does not declare" + )) + }) +} + +/// The request for `row` of page shard `child`, if the row is one a query of +/// that table can select. A pointer is a claim like any other. +fn locate( + manifest: &ShardManifest, + geometry: &transparent_shard::layout::Geometry, + shard_id: u64, + child: u8, + row: u64, +) -> Result { + let page = page_child(manifest, shard_id, child)?; + if page.page_segments.is_empty() { + return Err(SyncError::Content(format!( + "shard {shard_id} points into page shard {child}, which holds no rows" + ))); + } + let space = geometry.page_rows * page.page_segments.len() as u64; + let (_, within) = transparent_shard::layout::split_row(row, geometry.page_rows); + if row >= space || within >= page.query_rows() { + return Err(SyncError::Content(format!( + "shard {shard_id} page {row} is outside page shard {child}'s table" + ))); + } + Ok(RowRequest { + child: u32::from(child), + row, + }) +} + +impl Walker<'_> { + fn child(&self, child: u8) -> Result<&PageChild, SyncError> { + page_child(self.manifest, self.entry.shard_id, child) + } + + /// The request for `row` of page shard `child`; see [`locate`]. + fn locate(&self, child: u8, row: u64) -> Result { + locate( + self.manifest, + self.geometry, + self.entry.shard_id, + child, + row, + ) + } + + /// Checks a run's first fragment, read at `cursor`, before the walk + /// relies on it: every fragment it declares must be a row a page query + /// can select, or the run is not located at all. Then its link: the + /// older-run count its chain promised (see [`check_head`]) and a run to + /// link to that starts at a row of an earlier, declared page shard. A + /// link that fails any of these still leaves this run located, so the + /// walk owes the queries for its fragments not yet held, of `held`, and + /// one in the link's place unless the honest walk would not follow it, + /// all as dummies: the round the honest walk would send. + fn check_located( + &self, + store: &S, + script: &[u8], + head: &PageEntry, + cursor: &RunCursor, + held: usize, + ) -> Result<(), Failure> { + let last = u64::from(cursor.row) + u64::from(head.fragment_count.max(1)) - 1; + self.locate(cursor.child, last)?; + let link = check_head(head, cursor).and_then(|()| match head.link.previous() { + Some((child, row)) => self.locate(child, u64::from(row)).map(|_| ()), + None => Ok(()), + }); + let Err(error) = link else { + return Ok(()); + }; + let link_query = self.owed_link_query(store, script, head, cursor)?; + Err(Failure { + error, + child: cursor.child, + dummies: head.fragment_count.saturating_sub(held.max(1) as u32) + link_query, + }) + } + + /// Whether the honest walk would query a head after the run at `cursor`. + /// + /// It would not when the chain already promised this run was the oldest, + /// nor when this script's previous run is cached: a catch-up stops there + /// and sends nothing for it. When the link still names an earlier, + /// declared page shard (only its row or its older-run count is bad), that + /// is the page shard the honest walk would look up in the cache. Otherwise + /// the previous run is taken to be in the page shard just below. Either + /// way a bad link must not add a query the honest walk omits. + fn owed_link_query( + &self, + store: &S, + script: &[u8], + head: &PageEntry, + cursor: &RunCursor, + ) -> Result { + if cursor.older_runs == Some(0) { + return Ok(0); + } + let previous = match head.link.previous() { + Some((child, _)) if child < cursor.child && self.child(child).is_ok() => child, + _ => match cursor.child.checked_sub(1) { + Some(below) => below, + None => return Ok(1), + }, + }; + Ok(u32::from(self.cached(store, script, previous)?.is_none())) + } + + /// One round's requests for a walk, and the history cached below its + /// run when the walk will stop there. + fn plan( + &self, + store: &S, + walk: &Walk, + ) -> Result<(Vec, Option), SyncError> { + let cursor = walk.item.cursor; + let Some(head) = &walk.head else { + return Ok(( + vec![self.locate(cursor.child, u64::from(cursor.row))?], + None, + )); + }; + let mut requests = Vec::new(); + for ordinal in walk.item.fragments.len() as u32..head.fragment_count { + let row = u64::from(cursor.row) + u64::from(ordinal); + requests.push(self.locate(cursor.child, row)?); + } + let mut older = None; + if let Some((child, row)) = head.link.previous() { + older = self + .cached(store, &walk.item.script, child)? + .map(|events| (child, events)); + if older.is_none() { + requests.push(self.locate(child, u64::from(row))?); + } + } + Ok((requests, older)) + } + + /// The history cached for `script` from its run in page shard `child` + /// back to the start of the directory shard, when that page shard is + /// sealed. Each cached run holds only its own events and names the cached + /// run below it, so the history is put back together by following those + /// names; a missing run anywhere along the way is a miss. + fn cached( + &self, + store: &S, + script: &[u8], + child: u8, + ) -> Result>, SyncError> { + if !self.child(child)?.sealed { + return Ok(None); + } + let corrupt = |why: &str| SyncError::Store(StoreError::Corrupt(why.into())); + let mut index = usize::from(child); + let mut runs = Vec::new(); + loop { + let Some(run) = store.cached_run(&self.identities[index], script)? else { + return Ok(None); + }; + if run.events.is_empty() { + return Err(corrupt("a cached run holds no events")); + } + runs.push(run.events); + let Some(older) = run.older else { break }; + // The identity of a page shard covers every older one, so the run + // below must be in an earlier page shard of this directory shard. + index = self.identities[..index] + .iter() + .position(|identity| *identity == older) + .ok_or_else(|| { + corrupt("a cached run names a run outside the page shards below it") + })?; + } + let mut history: Vec = Vec::new(); + for run in runs.into_iter().rev() { + if let (Some(last), Some(first)) = (history.last(), run.first()) { + if last.sort_key() >= first.sort_key() { + return Err(corrupt("cached runs are not in order")); + } + } + history.extend(run); + } + Ok(Some(history)) + } + + /// The one fragment of `script`'s run in page shard `child` with this + /// ordinal, among every segment's row. Exactly one may claim it. + fn fragment( + &self, + script: &[u8], + child: u8, + answers: &[Vec], + ordinal: u32, + count: Option, + ) -> Result { + let shard_id = self.entry.shard_id; + let wanted = transparent_shard::script_tag(&self.salts[usize::from(child)], script); + // A packed row also carries other scripts' histories. Those are + // discarded, and deliberately not used: two of a wallet's own scripts + // sharing a row still cost two fetches, so the request transcript does + // not depend on which scripts share a row. + let mut found = None; + for raw in answers { + let decoded = transparent_shard::page_row::decode_page_row(raw).map_err(|error| { + SyncError::Content(format!("shard {shard_id} page shard {child}: {error}")) + })?; + for candidate in decoded { + if candidate.tag != wanted || candidate.ordinal != ordinal { + continue; + } + if count.is_some_and(|count| candidate.fragment_count != count) { + return Err(SyncError::Content(format!( + "shard {shard_id} page shard {child}: fragment {ordinal} disagrees with \ + its run's fragment count" + ))); + } + if found.is_some() { + return Err(SyncError::Content(format!( + "shard {shard_id} page shard {child} holds fragment {ordinal} twice" + ))); + } + found = Some(candidate); + } + } + found.ok_or_else(|| { + SyncError::Content(format!( + "shard {shard_id} page shard {child}: the row a link located does not hold \ + fragment {ordinal} of the run" + )) + }) + } + + /// Takes one walk through one round's replies. + fn advance( + &self, + store: &mut S, + walk: &mut Walk, + answers: &[Reply], + older: Option, + ) -> Result<(), Failure> { + let cursor = walk.item.cursor; + let script = walk.item.script.clone(); + if walk.head.is_none() { + let [answer] = answers else { + return Err(SyncError::Invalid("a round lost an answer".into()).into()); + }; + let head = self.fragment(&script, cursor.child, answer, 0, None)?; + self.check_located(store, &script, &head, &cursor, 1)?; + walk.item.fragments.push(encode_fragment(&head)?); + walk.head = Some(head); + return Ok(self.commit( + store, + &script, + Vec::new(), + Some(walk.item.clone()), + None, + Vec::new(), + )?); + } + let head = walk.head.as_ref().expect("head decoded above").clone(); + let count = head.fragment_count as usize; + if walk.item.fragments.len() > count { + return Err(SyncError::Invalid("a run has too many fragments".into()).into()); + } + let remaining = count - walk.item.fragments.len(); + let (answers, next) = answers.split_at(remaining.min(answers.len())); + if next.len() > 1 { + return Err(SyncError::Invalid("a round has extra answers".into()).into()); + } + // The run's fragments this round asked for and a budget left unsent: + // what the walk still owes if this run fails. + let unsent = (remaining - answers.len()) as u32; + let failing = |error: SyncError| Failure { + error, + child: cursor.child, + dummies: unsent, + }; + let mut boundary = validate_page_boundary(None, &head.events).map_err(failing)?; + let mut run = head.events.clone(); + // Recheck saved boundaries before using them after a restart. + for (ordinal, raw) in walk.item.fragments.iter().enumerate().skip(1) { + let fragment = self + .fragment( + &script, + cursor.child, + std::slice::from_ref(raw), + ordinal as u32, + Some(head.fragment_count), + ) + .map_err(failing)?; + boundary = + validate_page_boundary(Some(&boundary), &fragment.events).map_err(failing)?; + run.extend(fragment.events); + } + for answer in answers { + let ordinal = walk.item.fragments.len() as u32; + let fragment = self + .fragment( + &script, + cursor.child, + answer, + ordinal, + Some(head.fragment_count), + ) + .map_err(failing)?; + boundary = + validate_page_boundary(Some(&boundary), &fragment.events).map_err(failing)?; + run.extend(fragment.events.iter().copied()); + walk.item.fragments.push(encode_fragment(&fragment)?); + } + if walk.item.fragments.len() < count { + return Ok(self.commit( + store, + &script, + Vec::new(), + Some(walk.item.clone()), + None, + Vec::new(), + )?); + } + check_run(self.child(cursor.child)?, &run, &walk.item.carried)?; + walk.walked.push((cursor.child, run.clone())); + match head.link.previous() { + None => Ok(self.finish(store, walk, run, None)?), + Some(_) if older.is_some() => Ok(self.finish(store, walk, run, older)?), + Some((child, row)) => { + let mut following = run.clone(); + following.extend(walk.item.carried.iter().copied()); + following.truncate(2); + let mut progressed = walk.item.clone(); + progressed.cursor = RunCursor { + child, + row, + older_runs: Some(head.link.older_runs - 1), + }; + progressed.carried = following; + progressed.fragments.clear(); + // A bad head for the next run fails the walk, but not the run + // already checked: that is committed, and the walk resumes at + // the next run's head. + let mut failure = None; + let mut next_head = None; + if let [answer] = next { + let checked = self + .fragment(&script, child, answer, 0, None) + .map_err(Failure::from) + .and_then(|head| { + self.check_located(store, &script, &head, &progressed.cursor, 1)?; + Ok(head) + }); + match checked { + Ok(head) => { + progressed.fragments.push(encode_fragment(&head)?); + next_head = Some(head); + } + Err(error) => failure = Some(error), + } + } + progressed.attempts += 1; + self.commit( + store, + &script, + run, + Some(progressed.clone()), + None, + Vec::new(), + )?; + walk.item = progressed; + walk.head = next_head; + failure.map_or(Ok(()), Err) + } + } + } + + /// Takes a walk whose run is read in full through to its end, with no + /// replies: what a stopped round still finishes. + fn complete(&self, store: &mut S, walk: &mut Walk) -> Result<(), Failure> { + let older = match walk.head.as_ref().and_then(|head| head.link.previous()) { + Some((child, _)) => self + .cached(store, &walk.item.script, child)? + .map(|events| (child, events)), + None => None, + }; + self.advance(store, walk, &[], older) + } + + /// Ends a walk after `failure`. A content failure is counted against the + /// walk's entry, which keeps its pending item for a later sync, and the + /// walk sends the dummies it owes in the next round, if any; any other + /// failure is the call's. + fn fail( + &self, + store: &mut S, + walk: &mut Walk, + failures: &mut Failures, + failure: Failure, + ) -> Result<(), SyncError> { + let Failure { + error, + child, + dummies, + } = failure; + let SyncError::Content(_) = error else { + return Err(error); + }; + let entry = self.entry; + commit_bounded( + store, + self.target_anchor, + ShardCommit { + source_anchor: None, + shard_id: entry.shard_id, + revision_digest: entry.manifest_digest.clone(), + sealed: entry.sealed, + start_height: entry.start_height, + end_height: entry.end_height, + terminal_block_hash: entry.terminal_block_hash.clone(), + failed_entries: vec![walk.item.script.clone()], + ..Default::default() + }, + )?; + record_failure( + store, + &mut failures.entries, + entry.shard_id, + &walk.item.script, + )?; + if dummies > 0 { + walk.dummies = Some((child, dummies)); + failures + .owed + .push((entry.shard_id, walk.item.script.clone(), child, dummies)); + } else { + walk.done = true; + } + Ok(()) + } + + /// Ends a walk: `run` is the last run read, `older` the page shard and + /// history cached below it, if the walk stopped at a cached run. Fills the + /// run cache for every run this call read in a sealed page shard: each + /// with its own events and the name of the cached run below it. + fn finish( + &self, + store: &mut S, + walk: &mut Walk, + run: Vec, + older: Option, + ) -> Result<(), SyncError> { + let (mut below, older) = match older { + Some((child, events)) => (Some(self.identities[usize::from(child)].clone()), events), + None => (None, Vec::new()), + }; + let newer = run.first().or(walk.item.carried.first()); + if let (Some(last), Some(first)) = (older.last(), newer) { + if last.sort_key() >= first.sort_key() { + return Err(SyncError::Content( + "cached history is not older than the run that links to it".into(), + )); + } + } + let mut cached_runs = Vec::new(); + for (child, events) in walk.walked.iter().rev() { + // Only the newest run can be in the open page shard, so every run + // below a cached one is cached too. + if !self.child(*child)?.sealed { + break; + } + let identity = self.identities[usize::from(*child)].clone(); + cached_runs.push(CachedRun { + child_digest: identity.clone(), + script: walk.item.script.clone(), + shard_id: self.entry.shard_id, + events: events.clone(), + older: below.replace(identity), + }); + } + let mut events = older; + events.extend(run); + let script = walk.item.script.clone(); + self.commit(store, &script, events, None, walk.item.id, cached_runs)?; + walk.done = true; + Ok(()) + } + + /// Commits a step of one walk: the events it read, and either the walk's + /// next state or its end, with the script's coverage. + fn commit( + &self, + store: &mut S, + script: &[u8], + events: Vec, + progressed: Option, + finished: Option, + cached_runs: Vec, + ) -> Result<(), SyncError> { + let entry = self.entry; + let done = progressed.is_none(); + commit_bounded( + store, + self.target_anchor, + ShardCommit { + source_anchor: None, + shard_id: entry.shard_id, + revision_digest: entry.manifest_digest.clone(), + sealed: entry.sealed, + start_height: entry.start_height, + end_height: entry.end_height, + terminal_block_hash: entry.terminal_block_hash.clone(), + events: events + .into_iter() + .map(|event| StoredEvent { + script: script.to_vec(), + event, + shard_id: entry.shard_id, + revision_digest: entry.manifest_digest.clone(), + }) + .collect(), + covered_scripts: if done { + vec![script.to_vec()] + } else { + Vec::new() + }, + pending_upsert: progressed.into_iter().collect(), + pending_complete: finished.into_iter().collect(), + cached_runs, + failed_entries: Vec::new(), + }, + )?; + Ok(()) + } +} + +/// Checks a run's first fragment against the cursor that located it. +/// +/// Its `older_runs` must be the one fewer the newer run promised — which is +/// what makes a chain cut short, or one that carries on past where it should +/// end, a refusal rather than a shorter or longer history. It may not claim +/// more older runs than there are earlier page shards, and its link must +/// point to an earlier page shard: never this one, never a later one. +fn check_head(head: &PageEntry, cursor: &RunCursor) -> Result<(), SyncError> { + let link = head.link; + match cursor.older_runs { + Some(expected) if link.older_runs != expected => { + return Err(SyncError::Content(format!( + "a run in page shard {} reports {} older runs where its chain promised {expected}", + cursor.child, link.older_runs + ))) + } + _ if link.older_runs > cursor.child => { + return Err(SyncError::Content(format!( + "a run in page shard {} reports {} older runs", + cursor.child, link.older_runs + ))) + } + _ => {} + } + if let Some((child, _)) = link.previous() { + if child >= cursor.child { + return Err(SyncError::Content(format!( + "a run in page shard {} links to page shard {child}, which is not earlier", + cursor.child + ))); + } + } + Ok(()) +} + +/// Checks a run against the two events after it and its page shard. +/// +/// Events strictly increase from the run into `carried`. Each of the run's +/// events was pushed out when the event two places after it arrived, into the +/// page shard open then, so that event's height lies in this page shard's +/// range. A run placed in any other page shard fails here. +fn check_run( + child: &PageChild, + run: &[TransparentEvent], + carried: &[TransparentEvent], +) -> Result<(), SyncError> { + if run.is_empty() || carried.len() != transparent_shard::INLINE_EVENTS as usize { + return Err(SyncError::Content(format!( + "a run in page shard {} cannot be checked", + child.index + ))); + } + let following: Vec<&TransparentEvent> = run.iter().chain(carried).collect(); + if following + .windows(2) + .any(|pair| pair[0].sort_key() >= pair[1].sort_key()) + { + return Err(SyncError::Content(format!( + "a run in page shard {} is not older than the history after it", + child.index + ))); + } + for (index, _) in run.iter().enumerate() { + let height = event_height(following[index + 2]); + if height < child.start_height || height > child.end_height { + return Err(SyncError::Content(format!( + "an event in page shard {} was pushed out at height {height}, when that page \ + shard was not open", + child.index + ))); + } + } + Ok(()) +} + +fn validate_page_boundary( + previous: Option<&crate::store::PageBoundary>, + events: &[transparent_events::TransparentEvent], +) -> Result { + let first = events + .first() + .ok_or_else(|| SyncError::Content("empty fragment".into()))?; + if let Some(previous) = previous { + if previous.last_event.sort_key() > first.sort_key() + || previous.event_bytes as usize + + transparent_shard::compact::event_len(first, Some(&previous.last_event)) + <= transparent_shard::packing::FRAGMENT_PAYLOAD + { + return Err(SyncError::Content("noncanonical fragment boundary".into())); + } + } + Ok(crate::store::PageBoundary { + event_bytes: transparent_shard::compact::encoded_len(events) as u32, + last_event: *events.last().expect("nonempty fragment"), + }) +} + +/// Fetches the published setup for every segment of a shard's table, once, +/// reusing what the store holds under the exact set, revision and segment. +/// +/// Each segment publishes its own `c1`, so each is opened separately; a shard +/// with one segment — the ordinary case — costs exactly one setup per table. +#[allow(clippy::too_many_arguments)] +async fn open_table( + store: &mut S, + set_digest: &str, + shard_id: u64, + revision: &str, + table: Table, + segments: u32, + client: &mut TableClient, + transport: &mut impl ShardTransport, + charges: &mut ByteCharges, + limits: Option<&WorkLimits>, +) -> Result { + if segments == 0 { + return Err(SyncError::Invalid(format!( + "shard {shard_id} declares no {} segments", + table.name() + ))); + } + for segment in 0..segments { + if client.is_open(revision, table, segment) { + continue; + } + let key = SetupKey { + set_digest: set_digest.to_string(), + revision_digest: revision.to_string(), + table, + segment, + }; + let (params, digest) = match store.setup(&key)? { + Some(blob) => (blob.public_params_base64, blob.public_params_sha256), + None => { + if limits + .and_then(|limits| budget_stopped(charges, limits)) + .is_some() + { + return Ok(false); + } + let (raw, cost) = transport + .setup(shard_id, revision, table, segment) + .await + .map_err(|error| SyncError::from(classify_transport(error)))?; + charges.add_setup(table, cost); + let parsed: serde_json::Value = serde_json::from_slice(&raw) + .map_err(|error| SyncError::Transport(error.to_string()))?; + let params = parsed["public_params"] + .as_str() + .ok_or_else(|| SyncError::Invalid("setup has no public_params".into()))? + .to_string(); + let digest = parsed["public_params_sha256"] + .as_str() + .ok_or_else(|| SyncError::Invalid("setup has no digest".into()))? + .to_string(); + (params, digest) + } + }; + client.open_segment(revision, table, segment, ¶ms, &digest)?; + // Kept only after the client verified it against its digest. + store.put_setup( + &key, + &SetupBlob { + public_params_base64: params, + public_params_sha256: digest, + }, + )?; + } + Ok(true) +} + +fn check_target_hash(map: &ShardMap, target: &Anchor) -> Result<(), SyncError> { + if map + .shard_for_height(target.height) + .is_some_and(|s| s.end_height == target.height && s.terminal_block_hash != target.hash) + { + return Err(SyncError::Invalid( + "publication endpoint disagrees with wallet-accepted target hash".into(), + )); + } + Ok(()) +} + +/// Commits only the wallet-accepted prefix while retaining the publication identity. +fn commit_bounded( + store: &mut S, + target: &Anchor, + mut commit: ShardCommit, +) -> Result { + let source = Anchor { + height: commit.end_height, + hash: commit.terminal_block_hash.clone(), + }; + for event in commit + .events + .iter() + .map(|e| &e.event) + .chain(commit.pending_upsert.iter().flat_map(|p| p.carried.iter())) + { + let height = event_height(event); + if height < commit.start_height || height > source.height { + return Err(SyncError::Content( + "event lies outside its published shard".into(), + )); + } + } + commit.source_anchor = Some(source); + if commit.end_height >= target.height { + commit.end_height = target.height; + commit.terminal_block_hash = target.hash.clone(); + } + // Events above the target are not kept. A walk's carried events are not + // history but what it checks the next run against, so they stay whole. + commit + .events + .retain(|e| event_height(&e.event) <= target.height); + // What a reply carried contradicting what the store holds is the reply's + // content failure, not the store's. + store.commit_shard(commit).map_err(|error| match error { + StoreError::Contradiction(why) => SyncError::Content(format!("contradiction: {why}")), + other => SyncError::Store(other), + }) +} + +/// A checked fragment, saved as a page row holding only it. +fn encode_fragment(fragment: &PageEntry) -> Result, SyncError> { + Ok(transparent_shard::page_row::encode_page_row( + std::slice::from_ref(fragment), + )?) +} + +/// A sync's content failures so far: the entries, and the dummy queries +/// failed walks still owe. A shard read retried after an overload sends what +/// is still owed before the walk ends. +#[derive(Default)] +struct Failures { + entries: Vec, + /// Shard id, script, page shard and the number of dummies owed. + owed: Vec<(u64, Vec, u8, u32)>, +} + +impl Failures { + /// The dummies the walk of `script`'s entry in `shard_id` still owes. + fn owes(&self, shard_id: u64, script: &[u8]) -> Option<(u8, u32)> { + self.owed + .iter() + .find(|(shard, owner, ..)| *shard == shard_id && owner == script) + .map(|&(_, _, child, count)| (child, count)) + } + + /// Records that the dummies `owes` names were sent. + fn settle(&mut self, shard_id: u64, script: &[u8]) { + self.owed + .retain(|(shard, owner, ..)| !(*shard == shard_id && owner == script)); + } +} + +/// Whether `script`'s entry in shard `shard_id` is held out of this sync: +/// it failed on content earlier in the sync, or has failed +/// [`MAX_CONTENT_FAILURES`] times. A held entry's replies are not read; it +/// is queried again only by a shard read retried after an overload, which +/// asks what the first attempt asked. +fn held( + store: &S, + failed: &[FailedEntry], + shard_id: u64, + script: &[u8], +) -> Result { + Ok(failed + .iter() + .any(|entry| entry.shard_id == shard_id && entry.script == script) + || store.entry_failures(shard_id, script)? >= MAX_CONTENT_FAILURES) +} + +/// [`held`], listing an entry held for its failure count in the report the +/// first time this sync meets it. +fn hold( + store: &S, + failed: &mut Vec, + shard_id: u64, + script: &[u8], +) -> Result { + if failed + .iter() + .any(|entry| entry.shard_id == shard_id && entry.script == script) + { + return Ok(true); + } + let failures = store.entry_failures(shard_id, script)?; + if failures < MAX_CONTENT_FAILURES { + return Ok(false); + } + failed.push(FailedEntry { + shard_id, + script: script.to_vec(), + failures, + }); + Ok(true) +} + +/// Lists an entry whose failure a commit just counted, with the store's count. +fn record_failure( + store: &S, + failed: &mut Vec, + shard_id: u64, + script: &[u8], +) -> Result<(), SyncError> { + let failures = store.entry_failures(shard_id, script)?; + match failed + .iter_mut() + .find(|entry| entry.shard_id == shard_id && entry.script == script) + { + Some(entry) => entry.failures = failures, + None => failed.push(FailedEntry { + shard_id, + script: script.to_vec(), + failures, + }), + } + Ok(()) +} + +pub(crate) fn event_height(event: &transparent_events::TransparentEvent) -> u64 { + match event { + transparent_events::TransparentEvent::Receive(e) => u64::from(e.height), + transparent_events::TransparentEvent::Spend(e) => u64::from(e.height), + } +} + +fn accepted_at(chain: &impl ChainView, map: &ShardMap, height: u64) -> Result { + let hash = chain.hash_at(height).or_else(|| { + map.shards.iter().find_map(|s| { + if s.end_height == height { + Some(s.terminal_block_hash.clone()) + } else if s.start_height.checked_sub(1) == Some(height) { + Some(s.parent_block_hash.clone()) + } else { + None + } + }) + }); + match hash { + Some(hash) if chain.is_accepted(height, &hash) == Acceptance::Accepted => { + Ok(Anchor { height, hash }) + } + _ => Err(SyncError::Invalid(format!( + "wallet has no accepted rollback hash at {height}" + ))), + } +} + +/// The report of a sync stopped because the wallet's chain cannot confirm a +/// block at `height`: whatever was already rolled back, and nothing read. +#[allow(clippy::too_many_arguments)] +fn chain_unknown( + store: &S, + map: &ShardMap, + target_anchor: &Anchor, + charges: ByteCharges, + height: u64, + rolled_back_to: Option, + replaced_revisions: Vec, + first_commit: u64, +) -> Result { + let (covered, settled) = coverage_summary(store, map, target_anchor.height)?; + Ok(SyncReport { + ledger: store.ledger()?, + charges, + matched_shards: Vec::new(), + unproductive_matches: 0, + covered_through: covered, + settled_through: settled, + provisional: provisional_of(store, map)?, + map_refreshes: 0, + completion: Completion::Incomplete { + reason: IncompleteReason::ChainUnknown { height }, + pending: store.pending()?.len(), + }, + rolled_back_to, + replaced_revisions, + scripts_added: 0, + commits: store.last_commit()? - first_commit, + failed_entries: Vec::new(), + }) +} + +fn incomplete_at( + store: &S, + reason: IncompleteReason, +) -> Result { + Ok(SyncReport { + ledger: store.ledger()?, + charges: ByteCharges::default(), + matched_shards: vec![], + unproductive_matches: 0, + covered_through: 0, + settled_through: 0, + provisional: vec![], + map_refreshes: 0, + completion: Completion::Incomplete { + reason, + pending: store.pending()?.len(), + }, + rolled_back_to: None, + replaced_revisions: vec![], + scripts_added: 0, + commits: 0, + failed_entries: Vec::new(), + }) +} diff --git a/crates/z-transparent-pir-v2/src/sync_ahead.rs b/crates/z-transparent-pir-v2/src/sync_ahead.rs new file mode 100644 index 0000000..2c21ae0 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/sync_ahead.rs @@ -0,0 +1,497 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! Cross-shard request concurrency: the requests of a pass, sent together. +//! +//! The sequential walk reads one shard at a time: its filter, its manifest, its +//! directory setup, its directory queries, its pages. Each request waits for +//! the one before, so a restore spanning several matched shards pays one round +//! trip per request. Most of those requests do not depend on each other. Once +//! every uncached filter is local, which shards match and which scripts matched +//! them is known before any private request, and so are the manifests and +//! directory setups the walk will ask for. Once the manifests are verified, so +//! are the directory rows. +//! +//! [`look_ahead`] computes exactly those requests, hands them to the transport +//! as two batches, and stashes the outcome of every one that was sent where the +//! walk will look for it: manifests and setups in [`Staged`], filters in +//! [`StagedFilters`], queries in each directory +//! [`TableClient`](crate::client::TableClient) with the secret that decodes +//! them. The walk then runs unchanged. A stashed outcome is consumed at most +//! once, at the point where the sequential walk would have made that request, +//! and a refusal or failure is handled there exactly as if the request had just +//! been made: a withdrawn revision refreshes the map, an overload backs off and +//! retries within the same attempt budget, anything else fails the sync with +//! the store as the sequential walk would have left it. An unsent request is +//! made by the walk in its place. The walk is still what verifies, commits, +//! retries and refreshes, so its semantics are the sequential walk's. +//! +//! # What stays the same +//! +//! - Which requests are made, per shard and per table: the lookahead derives +//! them from the same filter bytes, script lists, choice table and setup +//! cache the walk would use, and the walk consumes rather than repeats them. +//! No query is skipped because another response mentioned a script. +//! - Charges: a stashed reply is charged when the walk consumes it, with what +//! the transport reported, so totals equal the sequential walk's. +//! - Commits: still per shard, in shard order, by the walk. +//! +//! # What changes +//! +//! When requests are sent. The service sees a pass's manifests and setups +//! together, then its directory queries together, instead of interleaved with +//! commits. Page work is not looked ahead: which pages a script needs is known +//! only after its directory entry decodes. The chain walk batches each of its +//! own rounds instead, across every script it walks. +//! +//! Requests the walk ends up not using are extra on the wire. If the walk stops +//! or restarts before a shard it looked ahead to — an exhausted overload +//! budget, a withdrawn revision and the refresh after it, a hard error — the +//! requests already sent for later shards were seen by the service and never +//! used. Those shards matched, so the next sync asks the same of them; a +//! refresh re-walks the rest of the pass in sequence. Likewise, requests +//! already in flight when a batch meets its first failure complete, and those +//! for the same shard after a refused query go unused, because the walk +//! retries that shard from its first query. +//! +//! The lookahead stops at the first shard whose endpoint, filter or manifest +//! does not check out, which is where the walk would stop too. A budgeted sync +//! never looks ahead, because whether a later shard's queries are sent at all +//! depends on page work only the walk discovers. + +use super::{ + directory_choice, endpoint_of, held, match_filter, previous_digest_of, verify_manifest, + Clients, FailedEntry, ServiceGeometry, SyncError, +}; +use crate::adapters::{Acceptance, ChainView}; +use crate::client::Table; +use crate::store::{Anchor, SetupKey, WalletStore}; +use crate::transport::{BoxError, FilterSource, ShardReply, ShardRequest, ShardTransport}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use transparent_filter::{BlockHash, ShardMap, ShardMapEntry}; +use transparent_shard::build::candidate_rows; +use transparent_shard::manifest::ShardManifest; + +/// The outcome of a request made ahead of the walk: its bytes and what they +/// cost, or its refusal or failure. +type Answer = Result<(Vec, u64), BoxError>; + +/// A shard transport with replies already in hand. +pub(super) struct Staged<'a, T: ShardTransport> { + inner: &'a mut T, + manifests: HashMap<(u64, String), Answer>, + setups: HashMap<(u64, String, Table, u32), Answer>, +} + +impl<'a, T: ShardTransport> Staged<'a, T> { + pub(super) fn new(inner: &'a mut T) -> Self { + Self { + inner, + manifests: HashMap::new(), + setups: HashMap::new(), + } + } + + pub(super) fn clear(&mut self) { + self.manifests.clear(); + self.setups.clear(); + } +} + +impl ShardTransport for Staged<'_, T> { + async fn init(&mut self) -> Result<(Vec, u64), BoxError> { + self.inner.init().await + } + + async fn manifest( + &mut self, + shard_id: u64, + revision: &str, + ) -> Result<(Vec, u64), BoxError> { + match self.manifests.remove(&(shard_id, revision.to_string())) { + Some(reply) => reply, + None => self.inner.manifest(shard_id, revision).await, + } + } + + async fn setup( + &mut self, + shard_id: u64, + revision: &str, + table: Table, + segment: u32, + ) -> Result<(Vec, u64), BoxError> { + match self + .setups + .remove(&(shard_id, revision.to_string(), table, segment)) + { + Some(reply) => reply, + None => self.inner.setup(shard_id, revision, table, segment).await, + } + } + + async fn query( + &mut self, + shard_id: u64, + revision: &str, + table: Table, + body: &[u8], + ) -> Result, BoxError> { + self.inner.query(shard_id, revision, table, body).await + } + + fn concurrency(&self) -> usize { + self.inner.concurrency() + } + + async fn batch(&mut self, requests: &[ShardRequest<'_>]) -> Vec> { + self.inner.batch(requests).await + } +} + +/// A filter source with filters already in hand. +/// +/// Keyed by shard id alone, as [`FilterSource::filter`] is, so the stash is +/// cleared whenever the walk refreshes the map: a filter fetched for a +/// withdrawn revision must not be handed out for the one that replaced it. +pub(super) struct StagedFilters<'a, F: FilterSource> { + inner: &'a mut F, + filters: HashMap, +} + +impl<'a, F: FilterSource> StagedFilters<'a, F> { + pub(super) fn new(inner: &'a mut F) -> Self { + Self { + inner, + filters: HashMap::new(), + } + } + + pub(super) fn clear(&mut self) { + self.filters.clear(); + } +} + +impl FilterSource for StagedFilters<'_, F> { + fn uses_parents(&self) -> bool { + self.inner.uses_parents() + } + + async fn shard_map(&mut self) -> Result<(Vec, u64), BoxError> { + self.inner.shard_map().await + } + + async fn filter(&mut self, shard_id: u64) -> Result<(Vec, u64), BoxError> { + match self.filters.remove(&shard_id) { + Some(reply) => reply, + None => self.inner.filter(shard_id).await, + } + } + + fn prefetch(&mut self, shard_ids: &[u64]) { + self.inner.prefetch(shard_ids) + } + + async fn prepare_parents( + &mut self, + map: &ShardMap, + uncached: &[u64], + store: &mut dyn WalletStore, + ) -> Result { + self.inner.prepare_parents(map, uncached, store).await + } + + async fn parent_negative( + &mut self, + map: &ShardMap, + shard_id: u64, + scripts: &[Vec], + store: &mut dyn WalletStore, + ) -> Result<(bool, u64), BoxError> { + self.inner + .parent_negative(map, shard_id, scripts, store) + .await + } +} + +/// Drops everything stashed, so the walk requests afresh from here on. +pub(super) fn clear( + transport: &mut Staged<'_, T>, + filters: &mut StagedFilters<'_, F>, + clients: &mut Clients, +) { + transport.clear(); + filters.clear(); + for prepared in clients.prepared.values_mut() { + prepared.directory.clear_stash(); + } +} + +/// A shard the walk will query, and the scripts it will query it for. +struct Planned<'m> { + entry: &'m ShardMapEntry, + matched: Vec>, +} + +/// Sends the independent requests of one pass ahead of the walk. +/// +/// `order` is the walk's shard order and `work` its scripts per shard. Only a +/// store failure is an error: anything else ends the lookahead early and +/// leaves the rest to the walk, which meets the same condition in its own +/// place and handles it as it always has. +#[allow(clippy::too_many_arguments)] +pub(super) async fn look_ahead( + store: &S, + map: &ShardMap, + genesis: BlockHash, + order: &[usize], + work: &BTreeMap>>, + geometry: &ServiceGeometry, + clients: &mut Clients, + filters: &mut StagedFilters<'_, F>, + transport: &mut Staged<'_, T>, + target_anchor: &Anchor, + chain: &impl ChainView, + failed: &[FailedEntry], +) -> Result<(), SyncError> { + // Match every shard the walk will reach, as it would. Filters not in the + // store are taken from the source here and stashed for the walk, which + // still checks, caches and charges them. + let pending = store.pending()?; + let mut planned: Vec> = Vec::new(); + for &index in order { + let Some(entry) = map.shards.get(index) else { + break; + }; + let endpoint = endpoint_of(entry, target_anchor); + if chain.is_accepted(endpoint.height, &endpoint.hash) != Acceptance::Accepted { + break; + } + let scripts = work.get(&index).map(Vec::as_slice).unwrap_or_default(); + let bytes = match store.filter(&entry.manifest_digest, &entry.filter_hash)? { + Some(bytes) => bytes, + None => { + // Whatever came back is the walk's outcome for this fetch, a + // failure or bytes that do not hash to the map included: its + // own check of them is what starts a refresh. + let reply = filters.inner.filter(entry.shard_id).await; + let usable = match &reply { + Ok((bytes, _)) + if transparent_filter::filter_hash(bytes).to_display_hex() + == entry.filter_hash => + { + Some(bytes.clone()) + } + _ => None, + }; + filters.filters.insert(entry.shard_id, reply); + match usable { + Some(bytes) => bytes, + None => break, + } + } + }; + let Ok(matches) = match_filter(map, entry, genesis, scripts, &bytes) else { + break; + }; + if matches.is_empty() { + continue; + } + if transparent_shard::layout::by_name(&entry.geometry).is_none() { + break; + } + // Durable page work for this revision changes what the walk asks for + // first; leave the shard to it. + if pending + .iter() + .any(|p| p.shard_id == entry.shard_id && p.revision_digest == entry.manifest_digest) + { + continue; + } + // Entries the walk holds out of this sync are not queried. + let mut matched = Vec::with_capacity(matches.len()); + for &i in &matches { + if !held(store, failed, entry.shard_id, &scripts[i])? { + matched.push(scripts[i].clone()); + } + } + if matched.is_empty() { + continue; + } + planned.push(Planned { entry, matched }); + } + if planned.is_empty() { + return Ok(()); + } + + // First batch: every manifest, and every directory setup the walk would + // fetch because neither the client nor the store holds it. + let set_digest = store + .set_identity()? + .map(|identity| identity.digest()) + .unwrap_or_default(); + let mut requests: Vec> = Vec::new(); + for shard in &planned { + let entry = shard.entry; + let revision = entry.manifest_digest.as_str(); + requests.push(ShardRequest::Manifest { + shard_id: entry.shard_id, + revision, + }); + for segment in 0..entry.directory_segments { + let open = clients + .prepared + .get(&entry.geometry) + .is_some_and(|prepared| { + prepared + .directory + .is_open(revision, Table::Directory, segment) + }); + let key = SetupKey { + set_digest: set_digest.clone(), + revision_digest: revision.to_string(), + table: Table::Directory, + segment, + }; + if !open && store.setup(&key)?.is_none() { + requests.push(ShardRequest::Setup { + shard_id: entry.shard_id, + revision, + table: Table::Directory, + segment, + }); + } + } + } + let replies = transport.inner.batch(&requests).await; + let mut manifests: HashMap = HashMap::new(); + for (request, reply) in requests.iter().zip(replies) { + // A sent request's outcome is kept whatever it was: a refusal reaches + // the walk where the sequential walk would have met it, and is + // handled there. An unsent one is simply made by the walk. + let Some(reply) = reply else { + continue; + }; + match *request { + ShardRequest::Manifest { shard_id, revision } => { + if let (Ok((raw, _)), Some(shard)) = ( + &reply, + planned.iter().find(|p| p.entry.shard_id == shard_id), + ) { + let previous = previous_digest_of(map, shard_id); + if let Ok(manifest) = verify_manifest(raw, shard.entry, &previous, map) { + manifests.insert(shard_id, manifest); + } + } + transport + .manifests + .insert((shard_id, revision.to_string()), reply); + } + ShardRequest::Setup { + shard_id, + revision, + table, + segment, + } => { + transport + .setups + .insert((shard_id, revision.to_string(), table, segment), reply); + } + ShardRequest::Query { .. } => unreachable!("the first batch holds no queries"), + } + } + + // Second batch: the directory queries, in walk order, prepared by the + // clients that will decode them. Rows are chosen exactly as the walk + // chooses them: the published choice when there is a table, both + // candidates when there is not. + struct Ahead<'m> { + entry: &'m ShardMapEntry, + geometry: String, + row: usize, + query: crate::client::PreparedQuery, + } + let mut queries: Vec> = Vec::new(); + 'shards: for shard in &planned { + let entry = shard.entry; + let Some(manifest) = manifests.get(&entry.shard_id) else { + break; + }; + let Ok(choice) = directory_choice(manifest) else { + break; + }; + let Ok(prepared) = clients.prepare(&manifest.geometry, &geometry.geometries) else { + break; + }; + let Some(layout) = transparent_shard::layout::by_name(&entry.geometry) else { + break; + }; + let rows = layout.directory_rows * u64::from(entry.directory_segments); + for script in &shard.matched { + let candidates = candidate_rows(entry.shard_id, script, rows); + let queried: &[u64] = match &choice { + Some(table) => { + std::slice::from_ref(&candidates[table.choice(entry.shard_id, script)]) + } + None => &candidates, + }; + for &row in queried { + let (_, within) = transparent_shard::layout::split_row(row, layout.directory_rows); + let Ok(query) = prepared.directory.prepare( + &entry.manifest_digest, + Table::Directory, + within as usize, + ) else { + break 'shards; + }; + queries.push(Ahead { + entry, + geometry: manifest.geometry.clone(), + row: within as usize, + query, + }); + } + } + } + if queries.is_empty() { + return Ok(()); + } + let requests: Vec> = queries + .iter() + .map(|ahead| ShardRequest::Query { + shard_id: ahead.entry.shard_id, + revision: &ahead.entry.manifest_digest, + table: Table::Directory, + body: &ahead.query.body, + }) + .collect(); + let replies = transport.inner.batch(&requests).await; + drop(requests); + // A revision's stash must be a prefix of what the walk will ask for it. + // After an unsent query the walk prepares and sends that one and the rest + // itself; after a refused or failed one it starts the shard over, so + // nothing later of that revision would be used either. + let mut ended: HashSet<&str> = HashSet::new(); + for (ahead, reply) in queries.into_iter().zip(replies) { + let revision = ahead.entry.manifest_digest.as_str(); + if ended.contains(revision) { + continue; + } + let Some(reply) = reply else { + ended.insert(revision); + continue; + }; + if reply.is_err() { + ended.insert(revision); + } + if let Some(prepared) = clients.prepared.get_mut(&ahead.geometry) { + prepared.directory.stash( + revision, + Table::Directory, + ahead.row, + ahead.query, + reply.map(|(bytes, _)| bytes), + ); + } + } + Ok(()) +} diff --git a/crates/z-transparent-pir-v2/src/transport.rs b/crates/z-transparent-pir-v2/src/transport.rs new file mode 100644 index 0000000..1627e77 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/transport.rs @@ -0,0 +1,87 @@ +use crate::client::Table; +use crate::store::WalletStore; +pub use transparent_wallet::transport::{ + BoxError, ByteCharges, Overloaded, ShardReply, ShardRequest, StaleRevision, +}; +pub(crate) fn describe_error(e: &(dyn std::error::Error + 'static)) -> String { + e.to_string() +} +pub trait FilterSource { + fn uses_parents(&self) -> bool { + false + } + async fn shard_map(&mut self) -> Result<(Vec, u64), BoxError>; + async fn filter(&mut self, id: u64) -> Result<(Vec, u64), BoxError>; + fn prefetch(&mut self, _ids: &[u64]) {} + async fn prepare_parents( + &mut self, + _map: &transparent_filter::ShardMap, + _ids: &[u64], + _s: &mut dyn WalletStore, + ) -> Result { + Ok(0) + } + async fn parent_negative( + &mut self, + _map: &transparent_filter::ShardMap, + _id: u64, + _scripts: &[Vec], + _s: &mut dyn WalletStore, + ) -> Result<(bool, u64), BoxError> { + Ok((false, 0)) + } +} +pub trait ShardTransport { + async fn init(&mut self) -> Result<(Vec, u64), BoxError>; + async fn manifest(&mut self, id: u64, rev: &str) -> Result<(Vec, u64), BoxError>; + async fn setup( + &mut self, + id: u64, + rev: &str, + table: Table, + segment: u32, + ) -> Result<(Vec, u64), BoxError>; + async fn query( + &mut self, + id: u64, + rev: &str, + table: Table, + body: &[u8], + ) -> Result, BoxError>; + fn concurrency(&self) -> usize { + 1 + } + async fn batch(&mut self, requests: &[ShardRequest<'_>]) -> Vec> { + let mut out = Vec::new(); + let mut failed = false; + for r in requests { + if failed { + out.push(None); + continue; + } + let result = match *r { + ShardRequest::Manifest { shard_id, revision } => { + self.manifest(shard_id, revision).await + } + ShardRequest::Setup { + shard_id, + revision, + table, + segment, + } => self.setup(shard_id, revision, table, segment).await, + ShardRequest::Query { + shard_id, + revision, + table, + body, + } => self.query(shard_id, revision, table, body).await.map(|b| { + let n = b.len() as u64; + (b, n) + }), + }; + failed = result.is_err(); + out.push(Some(result)); + } + out + } +} diff --git a/crates/z-transparent-pir-v2/src/types.rs b/crates/z-transparent-pir-v2/src/types.rs new file mode 100644 index 0000000..a4b0ca4 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/types.rs @@ -0,0 +1,147 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Table { + Directory, + Pages { child: u32 }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Operation { + Init, + Map, + Filter { + shard_id: u64, + }, + Manifest { + shard_id: u64, + revision: String, + }, + Setup { + shard_id: u64, + revision: String, + table: Table, + segment: u32, + }, + Query { + shard_id: u64, + revision: String, + table: Table, + body: String, + }, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Reply { + Data { body: String }, + Stale { map_digest_hint: Option }, + Overloaded { retry_after: Option }, + Misrouted, + Unavailable, + Refused, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Anchor { + pub height: u64, + pub hash: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Script { + pub script: String, + pub required_from: u64, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scope { + pub source_id: String, + pub generation: u64, + pub scope_id: String, + pub anchor: Anchor, + pub scripts: Vec